kernel/drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "sid.h"
  33 #include "atom.h"
  34 #include "si_blit_shaders.h"
  35 #include "clearstate_si.h"
  36 #include "radeon_ucode.h"
  37
  38
  39 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  40 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  41 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  42 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  43 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  44 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  45 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  46
  47 MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  48 MODULE_FIRMWARE("radeon/tahiti_me.bin");
  49 MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  50 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  51 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  52 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  53
  54 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  55 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  56 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  57 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  58 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  59 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  60 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  61
  62 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  63 MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  64 MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  65 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  66 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  67 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
  68
  69 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  70 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  71 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  72 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  73 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  74 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  75 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  76
  77 MODULE_FIRMWARE("radeon/verde_pfp.bin");
  78 MODULE_FIRMWARE("radeon/verde_me.bin");
  79 MODULE_FIRMWARE("radeon/verde_ce.bin");
  80 MODULE_FIRMWARE("radeon/verde_mc.bin");
  81 MODULE_FIRMWARE("radeon/verde_rlc.bin");
  82 MODULE_FIRMWARE("radeon/verde_smc.bin");
  83
  84 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  85 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  86 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  87 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  88 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  89 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  90 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  91
  92 MODULE_FIRMWARE("radeon/oland_pfp.bin");
  93 MODULE_FIRMWARE("radeon/oland_me.bin");
  94 MODULE_FIRMWARE("radeon/oland_ce.bin");
  95 MODULE_FIRMWARE("radeon/oland_mc.bin");
  96 MODULE_FIRMWARE("radeon/oland_rlc.bin");
  97 MODULE_FIRMWARE("radeon/oland_smc.bin");
  98
  99 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 100 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 101 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 102 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 103 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 104 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 105 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 106
 107 MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 108 MODULE_FIRMWARE("radeon/hainan_me.bin");
 109 MODULE_FIRMWARE("radeon/hainan_ce.bin");
 110 MODULE_FIRMWARE("radeon/hainan_mc.bin");
 111 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 112 MODULE_FIRMWARE("radeon/hainan_smc.bin");
 113
 114 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 115 static void si_pcie_gen3_enable(struct radeon_device *rdev);
 116 static void si_program_aspm(struct radeon_device *rdev);
 117 extern void sumo_rlc_fini(struct radeon_device *rdev);
 118 extern int sumo_rlc_init(struct radeon_device *rdev);
 119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
 121 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 122 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 123 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 124 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
 125 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 126 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 127 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 128                                          bool enable);
 129 static void si_init_pg(struct radeon_device *rdev);
 130 static void si_init_cg(struct radeon_device *rdev);
 131 static void si_fini_pg(struct radeon_device *rdev);
 132 static void si_fini_cg(struct radeon_device *rdev);
 133 static void si_rlc_stop(struct radeon_device *rdev);
 134
 135 static const u32 verde_rlc_save_restore_register_list[] =
 136 {
 137         (0x8000 << 16) | (0x98f4 >> 2),
 138         0x00000000,
 139         (0x8040 << 16) | (0x98f4 >> 2),
 140         0x00000000,
 141         (0x8000 << 16) | (0xe80 >> 2),
 142         0x00000000,
 143         (0x8040 << 16) | (0xe80 >> 2),
 144         0x00000000,
 145         (0x8000 << 16) | (0x89bc >> 2),
 146         0x00000000,
 147         (0x8040 << 16) | (0x89bc >> 2),
 148         0x00000000,
 149         (0x8000 << 16) | (0x8c1c >> 2),
 150         0x00000000,
 151         (0x8040 << 16) | (0x8c1c >> 2),
 152         0x00000000,
 153         (0x9c00 << 16) | (0x98f0 >> 2),
 154         0x00000000,
 155         (0x9c00 << 16) | (0xe7c >> 2),
 156         0x00000000,
 157         (0x8000 << 16) | (0x9148 >> 2),
 158         0x00000000,
 159         (0x8040 << 16) | (0x9148 >> 2),
 160         0x00000000,
 161         (0x9c00 << 16) | (0x9150 >> 2),
 162         0x00000000,
 163         (0x9c00 << 16) | (0x897c >> 2),
 164         0x00000000,
 165         (0x9c00 << 16) | (0x8d8c >> 2),
 166         0x00000000,
 167         (0x9c00 << 16) | (0xac54 >> 2),
 168         0X00000000,
 169         0x3,
 170         (0x9c00 << 16) | (0x98f8 >> 2),
 171         0x00000000,
 172         (0x9c00 << 16) | (0x9910 >> 2),
 173         0x00000000,
 174         (0x9c00 << 16) | (0x9914 >> 2),
 175         0x00000000,
 176         (0x9c00 << 16) | (0x9918 >> 2),
 177         0x00000000,
 178         (0x9c00 << 16) | (0x991c >> 2),
 179         0x00000000,
 180         (0x9c00 << 16) | (0x9920 >> 2),
 181         0x00000000,
 182         (0x9c00 << 16) | (0x9924 >> 2),
 183         0x00000000,
 184         (0x9c00 << 16) | (0x9928 >> 2),
 185         0x00000000,
 186         (0x9c00 << 16) | (0x992c >> 2),
 187         0x00000000,
 188         (0x9c00 << 16) | (0x9930 >> 2),
 189         0x00000000,
 190         (0x9c00 << 16) | (0x9934 >> 2),
 191         0x00000000,
 192         (0x9c00 << 16) | (0x9938 >> 2),
 193         0x00000000,
 194         (0x9c00 << 16) | (0x993c >> 2),
 195         0x00000000,
 196         (0x9c00 << 16) | (0x9940 >> 2),
 197         0x00000000,
 198         (0x9c00 << 16) | (0x9944 >> 2),
 199         0x00000000,
 200         (0x9c00 << 16) | (0x9948 >> 2),
 201         0x00000000,
 202         (0x9c00 << 16) | (0x994c >> 2),
 203         0x00000000,
 204         (0x9c00 << 16) | (0x9950 >> 2),
 205         0x00000000,
 206         (0x9c00 << 16) | (0x9954 >> 2),
 207         0x00000000,
 208         (0x9c00 << 16) | (0x9958 >> 2),
 209         0x00000000,
 210         (0x9c00 << 16) | (0x995c >> 2),
 211         0x00000000,
 212         (0x9c00 << 16) | (0x9960 >> 2),
 213         0x00000000,
 214         (0x9c00 << 16) | (0x9964 >> 2),
 215         0x00000000,
 216         (0x9c00 << 16) | (0x9968 >> 2),
 217         0x00000000,
 218         (0x9c00 << 16) | (0x996c >> 2),
 219         0x00000000,
 220         (0x9c00 << 16) | (0x9970 >> 2),
 221         0x00000000,
 222         (0x9c00 << 16) | (0x9974 >> 2),
 223         0x00000000,
 224         (0x9c00 << 16) | (0x9978 >> 2),
 225         0x00000000,
 226         (0x9c00 << 16) | (0x997c >> 2),
 227         0x00000000,
 228         (0x9c00 << 16) | (0x9980 >> 2),
 229         0x00000000,
 230         (0x9c00 << 16) | (0x9984 >> 2),
 231         0x00000000,
 232         (0x9c00 << 16) | (0x9988 >> 2),
 233         0x00000000,
 234         (0x9c00 << 16) | (0x998c >> 2),
 235         0x00000000,
 236         (0x9c00 << 16) | (0x8c00 >> 2),
 237         0x00000000,
 238         (0x9c00 << 16) | (0x8c14 >> 2),
 239         0x00000000,
 240         (0x9c00 << 16) | (0x8c04 >> 2),
 241         0x00000000,
 242         (0x9c00 << 16) | (0x8c08 >> 2),
 243         0x00000000,
 244         (0x8000 << 16) | (0x9b7c >> 2),
 245         0x00000000,
 246         (0x8040 << 16) | (0x9b7c >> 2),
 247         0x00000000,
 248         (0x8000 << 16) | (0xe84 >> 2),
 249         0x00000000,
 250         (0x8040 << 16) | (0xe84 >> 2),
 251         0x00000000,
 252         (0x8000 << 16) | (0x89c0 >> 2),
 253         0x00000000,
 254         (0x8040 << 16) | (0x89c0 >> 2),
 255         0x00000000,
 256         (0x8000 << 16) | (0x914c >> 2),
 257         0x00000000,
 258         (0x8040 << 16) | (0x914c >> 2),
 259         0x00000000,
 260         (0x8000 << 16) | (0x8c20 >> 2),
 261         0x00000000,
 262         (0x8040 << 16) | (0x8c20 >> 2),
 263         0x00000000,
 264         (0x8000 << 16) | (0x9354 >> 2),
 265         0x00000000,
 266         (0x8040 << 16) | (0x9354 >> 2),
 267         0x00000000,
 268         (0x9c00 << 16) | (0x9060 >> 2),
 269         0x00000000,
 270         (0x9c00 << 16) | (0x9364 >> 2),
 271         0x00000000,
 272         (0x9c00 << 16) | (0x9100 >> 2),
 273         0x00000000,
 274         (0x9c00 << 16) | (0x913c >> 2),
 275         0x00000000,
 276         (0x8000 << 16) | (0x90e0 >> 2),
 277         0x00000000,
 278         (0x8000 << 16) | (0x90e4 >> 2),
 279         0x00000000,
 280         (0x8000 << 16) | (0x90e8 >> 2),
 281         0x00000000,
 282         (0x8040 << 16) | (0x90e0 >> 2),
 283         0x00000000,
 284         (0x8040 << 16) | (0x90e4 >> 2),
 285         0x00000000,
 286         (0x8040 << 16) | (0x90e8 >> 2),
 287         0x00000000,
 288         (0x9c00 << 16) | (0x8bcc >> 2),
 289         0x00000000,
 290         (0x9c00 << 16) | (0x8b24 >> 2),
 291         0x00000000,
 292         (0x9c00 << 16) | (0x88c4 >> 2),
 293         0x00000000,
 294         (0x9c00 << 16) | (0x8e50 >> 2),
 295         0x00000000,
 296         (0x9c00 << 16) | (0x8c0c >> 2),
 297         0x00000000,
 298         (0x9c00 << 16) | (0x8e58 >> 2),
 299         0x00000000,
 300         (0x9c00 << 16) | (0x8e5c >> 2),
 301         0x00000000,
 302         (0x9c00 << 16) | (0x9508 >> 2),
 303         0x00000000,
 304         (0x9c00 << 16) | (0x950c >> 2),
 305         0x00000000,
 306         (0x9c00 << 16) | (0x9494 >> 2),
 307         0x00000000,
 308         (0x9c00 << 16) | (0xac0c >> 2),
 309         0x00000000,
 310         (0x9c00 << 16) | (0xac10 >> 2),
 311         0x00000000,
 312         (0x9c00 << 16) | (0xac14 >> 2),
 313         0x00000000,
 314         (0x9c00 << 16) | (0xae00 >> 2),
 315         0x00000000,
 316         (0x9c00 << 16) | (0xac08 >> 2),
 317         0x00000000,
 318         (0x9c00 << 16) | (0x88d4 >> 2),
 319         0x00000000,
 320         (0x9c00 << 16) | (0x88c8 >> 2),
 321         0x00000000,
 322         (0x9c00 << 16) | (0x88cc >> 2),
 323         0x00000000,
 324         (0x9c00 << 16) | (0x89b0 >> 2),
 325         0x00000000,
 326         (0x9c00 << 16) | (0x8b10 >> 2),
 327         0x00000000,
 328         (0x9c00 << 16) | (0x8a14 >> 2),
 329         0x00000000,
 330         (0x9c00 << 16) | (0x9830 >> 2),
 331         0x00000000,
 332         (0x9c00 << 16) | (0x9834 >> 2),
 333         0x00000000,
 334         (0x9c00 << 16) | (0x9838 >> 2),
 335         0x00000000,
 336         (0x9c00 << 16) | (0x9a10 >> 2),
 337         0x00000000,
 338         (0x8000 << 16) | (0x9870 >> 2),
 339         0x00000000,
 340         (0x8000 << 16) | (0x9874 >> 2),
 341         0x00000000,
 342         (0x8001 << 16) | (0x9870 >> 2),
 343         0x00000000,
 344         (0x8001 << 16) | (0x9874 >> 2),
 345         0x00000000,
 346         (0x8040 << 16) | (0x9870 >> 2),
 347         0x00000000,
 348         (0x8040 << 16) | (0x9874 >> 2),
 349         0x00000000,
 350         (0x8041 << 16) | (0x9870 >> 2),
 351         0x00000000,
 352         (0x8041 << 16) | (0x9874 >> 2),
 353         0x00000000,
 354         0x00000000
 355 };
 356
 357 static const u32 tahiti_golden_rlc_registers[] =
 358 {
 359         0xc424, 0xffffffff, 0x00601005,
 360         0xc47c, 0xffffffff, 0x10104040,
 361         0xc488, 0xffffffff, 0x0100000a,
 362         0xc314, 0xffffffff, 0x00000800,
 363         0xc30c, 0xffffffff, 0x800000f4,
 364         0xf4a8, 0xffffffff, 0x00000000
 365 };
 366
 367 static const u32 tahiti_golden_registers[] =
 368 {
 369         0x9a10, 0x00010000, 0x00018208,
 370         0x9830, 0xffffffff, 0x00000000,
 371         0x9834, 0xf00fffff, 0x00000400,
 372         0x9838, 0x0002021c, 0x00020200,
 373         0xc78, 0x00000080, 0x00000000,
 374         0xd030, 0x000300c0, 0x00800040,
 375         0xd830, 0x000300c0, 0x00800040,
 376         0x5bb0, 0x000000f0, 0x00000070,
 377         0x5bc0, 0x00200000, 0x50100000,
 378         0x7030, 0x31000311, 0x00000011,
 379         0x277c, 0x00000003, 0x000007ff,
 380         0x240c, 0x000007ff, 0x00000000,
 381         0x8a14, 0xf000001f, 0x00000007,
 382         0x8b24, 0xffffffff, 0x00ffffff,
 383         0x8b10, 0x0000ff0f, 0x00000000,
 384         0x28a4c, 0x07ffffff, 0x4e000000,
 385         0x28350, 0x3f3f3fff, 0x2a00126a,
 386         0x30, 0x000000ff, 0x0040,
 387         0x34, 0x00000040, 0x00004040,
 388         0x9100, 0x07ffffff, 0x03000000,
 389         0x8e88, 0x01ff1f3f, 0x00000000,
 390         0x8e84, 0x01ff1f3f, 0x00000000,
 391         0x9060, 0x0000007f, 0x00000020,
 392         0x9508, 0x00010000, 0x00010000,
 393         0xac14, 0x00000200, 0x000002fb,
 394         0xac10, 0xffffffff, 0x0000543b,
 395         0xac0c, 0xffffffff, 0xa9210876,
 396         0x88d0, 0xffffffff, 0x000fff40,
 397         0x88d4, 0x0000001f, 0x00000010,
 398         0x1410, 0x20000000, 0x20fffed8,
 399         0x15c0, 0x000c0fc0, 0x000c0400
 400 };
 401
 402 static const u32 tahiti_golden_registers2[] =
 403 {
 404         0xc64, 0x00000001, 0x00000001
 405 };
 406
 407 static const u32 pitcairn_golden_rlc_registers[] =
 408 {
 409         0xc424, 0xffffffff, 0x00601004,
 410         0xc47c, 0xffffffff, 0x10102020,
 411         0xc488, 0xffffffff, 0x01000020,
 412         0xc314, 0xffffffff, 0x00000800,
 413         0xc30c, 0xffffffff, 0x800000a4
 414 };
 415
 416 static const u32 pitcairn_golden_registers[] =
 417 {
 418         0x9a10, 0x00010000, 0x00018208,
 419         0x9830, 0xffffffff, 0x00000000,
 420         0x9834, 0xf00fffff, 0x00000400,
 421         0x9838, 0x0002021c, 0x00020200,
 422         0xc78, 0x00000080, 0x00000000,
 423         0xd030, 0x000300c0, 0x00800040,
 424         0xd830, 0x000300c0, 0x00800040,
 425         0x5bb0, 0x000000f0, 0x00000070,
 426         0x5bc0, 0x00200000, 0x50100000,
 427         0x7030, 0x31000311, 0x00000011,
 428         0x2ae4, 0x00073ffe, 0x000022a2,
 429         0x240c, 0x000007ff, 0x00000000,
 430         0x8a14, 0xf000001f, 0x00000007,
 431         0x8b24, 0xffffffff, 0x00ffffff,
 432         0x8b10, 0x0000ff0f, 0x00000000,
 433         0x28a4c, 0x07ffffff, 0x4e000000,
 434         0x28350, 0x3f3f3fff, 0x2a00126a,
 435         0x30, 0x000000ff, 0x0040,
 436         0x34, 0x00000040, 0x00004040,
 437         0x9100, 0x07ffffff, 0x03000000,
 438         0x9060, 0x0000007f, 0x00000020,
 439         0x9508, 0x00010000, 0x00010000,
 440         0xac14, 0x000003ff, 0x000000f7,
 441         0xac10, 0xffffffff, 0x00000000,
 442         0xac0c, 0xffffffff, 0x32761054,
 443         0x88d4, 0x0000001f, 0x00000010,
 444         0x15c0, 0x000c0fc0, 0x000c0400
 445 };
 446
 447 static const u32 verde_golden_rlc_registers[] =
 448 {
 449         0xc424, 0xffffffff, 0x033f1005,
 450         0xc47c, 0xffffffff, 0x10808020,
 451         0xc488, 0xffffffff, 0x00800008,
 452         0xc314, 0xffffffff, 0x00001000,
 453         0xc30c, 0xffffffff, 0x80010014
 454 };
 455
 456 static const u32 verde_golden_registers[] =
 457 {
 458         0x9a10, 0x00010000, 0x00018208,
 459         0x9830, 0xffffffff, 0x00000000,
 460         0x9834, 0xf00fffff, 0x00000400,
 461         0x9838, 0x0002021c, 0x00020200,
 462         0xc78, 0x00000080, 0x00000000,
 463         0xd030, 0x000300c0, 0x00800040,
 464         0xd030, 0x000300c0, 0x00800040,
 465         0xd830, 0x000300c0, 0x00800040,
 466         0xd830, 0x000300c0, 0x00800040,
 467         0x5bb0, 0x000000f0, 0x00000070,
 468         0x5bc0, 0x00200000, 0x50100000,
 469         0x7030, 0x31000311, 0x00000011,
 470         0x2ae4, 0x00073ffe, 0x000022a2,
 471         0x2ae4, 0x00073ffe, 0x000022a2,
 472         0x2ae4, 0x00073ffe, 0x000022a2,
 473         0x240c, 0x000007ff, 0x00000000,
 474         0x240c, 0x000007ff, 0x00000000,
 475         0x240c, 0x000007ff, 0x00000000,
 476         0x8a14, 0xf000001f, 0x00000007,
 477         0x8a14, 0xf000001f, 0x00000007,
 478         0x8a14, 0xf000001f, 0x00000007,
 479         0x8b24, 0xffffffff, 0x00ffffff,
 480         0x8b10, 0x0000ff0f, 0x00000000,
 481         0x28a4c, 0x07ffffff, 0x4e000000,
 482         0x28350, 0x3f3f3fff, 0x0000124a,
 483         0x28350, 0x3f3f3fff, 0x0000124a,
 484         0x28350, 0x3f3f3fff, 0x0000124a,
 485         0x30, 0x000000ff, 0x0040,
 486         0x34, 0x00000040, 0x00004040,
 487         0x9100, 0x07ffffff, 0x03000000,
 488         0x9100, 0x07ffffff, 0x03000000,
 489         0x8e88, 0x01ff1f3f, 0x00000000,
 490         0x8e88, 0x01ff1f3f, 0x00000000,
 491         0x8e88, 0x01ff1f3f, 0x00000000,
 492         0x8e84, 0x01ff1f3f, 0x00000000,
 493         0x8e84, 0x01ff1f3f, 0x00000000,
 494         0x8e84, 0x01ff1f3f, 0x00000000,
 495         0x9060, 0x0000007f, 0x00000020,
 496         0x9508, 0x00010000, 0x00010000,
 497         0xac14, 0x000003ff, 0x00000003,
 498         0xac14, 0x000003ff, 0x00000003,
 499         0xac14, 0x000003ff, 0x00000003,
 500         0xac10, 0xffffffff, 0x00000000,
 501         0xac10, 0xffffffff, 0x00000000,
 502         0xac10, 0xffffffff, 0x00000000,
 503         0xac0c, 0xffffffff, 0x00001032,
 504         0xac0c, 0xffffffff, 0x00001032,
 505         0xac0c, 0xffffffff, 0x00001032,
 506         0x88d4, 0x0000001f, 0x00000010,
 507         0x88d4, 0x0000001f, 0x00000010,
 508         0x88d4, 0x0000001f, 0x00000010,
 509         0x15c0, 0x000c0fc0, 0x000c0400
 510 };
 511
 512 static const u32 oland_golden_rlc_registers[] =
 513 {
 514         0xc424, 0xffffffff, 0x00601005,
 515         0xc47c, 0xffffffff, 0x10104040,
 516         0xc488, 0xffffffff, 0x0100000a,
 517         0xc314, 0xffffffff, 0x00000800,
 518         0xc30c, 0xffffffff, 0x800000f4
 519 };
 520
 521 static const u32 oland_golden_registers[] =
 522 {
 523         0x9a10, 0x00010000, 0x00018208,
 524         0x9830, 0xffffffff, 0x00000000,
 525         0x9834, 0xf00fffff, 0x00000400,
 526         0x9838, 0x0002021c, 0x00020200,
 527         0xc78, 0x00000080, 0x00000000,
 528         0xd030, 0x000300c0, 0x00800040,
 529         0xd830, 0x000300c0, 0x00800040,
 530         0x5bb0, 0x000000f0, 0x00000070,
 531         0x5bc0, 0x00200000, 0x50100000,
 532         0x7030, 0x31000311, 0x00000011,
 533         0x2ae4, 0x00073ffe, 0x000022a2,
 534         0x240c, 0x000007ff, 0x00000000,
 535         0x8a14, 0xf000001f, 0x00000007,
 536         0x8b24, 0xffffffff, 0x00ffffff,
 537         0x8b10, 0x0000ff0f, 0x00000000,
 538         0x28a4c, 0x07ffffff, 0x4e000000,
 539         0x28350, 0x3f3f3fff, 0x00000082,
 540         0x30, 0x000000ff, 0x0040,
 541         0x34, 0x00000040, 0x00004040,
 542         0x9100, 0x07ffffff, 0x03000000,
 543         0x9060, 0x0000007f, 0x00000020,
 544         0x9508, 0x00010000, 0x00010000,
 545         0xac14, 0x000003ff, 0x000000f3,
 546         0xac10, 0xffffffff, 0x00000000,
 547         0xac0c, 0xffffffff, 0x00003210,
 548         0x88d4, 0x0000001f, 0x00000010,
 549         0x15c0, 0x000c0fc0, 0x000c0400
 550 };
 551
 552 static const u32 hainan_golden_registers[] =
 553 {
 554         0x9a10, 0x00010000, 0x00018208,
 555         0x9830, 0xffffffff, 0x00000000,
 556         0x9834, 0xf00fffff, 0x00000400,
 557         0x9838, 0x0002021c, 0x00020200,
 558         0xd0c0, 0xff000fff, 0x00000100,
 559         0xd030, 0x000300c0, 0x00800040,
 560         0xd8c0, 0xff000fff, 0x00000100,
 561         0xd830, 0x000300c0, 0x00800040,
 562         0x2ae4, 0x00073ffe, 0x000022a2,
 563         0x240c, 0x000007ff, 0x00000000,
 564         0x8a14, 0xf000001f, 0x00000007,
 565         0x8b24, 0xffffffff, 0x00ffffff,
 566         0x8b10, 0x0000ff0f, 0x00000000,
 567         0x28a4c, 0x07ffffff, 0x4e000000,
 568         0x28350, 0x3f3f3fff, 0x00000000,
 569         0x30, 0x000000ff, 0x0040,
 570         0x34, 0x00000040, 0x00004040,
 571         0x9100, 0x03e00000, 0x03600000,
 572         0x9060, 0x0000007f, 0x00000020,
 573         0x9508, 0x00010000, 0x00010000,
 574         0xac14, 0x000003ff, 0x000000f1,
 575         0xac10, 0xffffffff, 0x00000000,
 576         0xac0c, 0xffffffff, 0x00003210,
 577         0x88d4, 0x0000001f, 0x00000010,
 578         0x15c0, 0x000c0fc0, 0x000c0400
 579 };
 580
 581 static const u32 hainan_golden_registers2[] =
 582 {
 583         0x98f8, 0xffffffff, 0x02010001
 584 };
 585
 586 static const u32 tahiti_mgcg_cgcg_init[] =
 587 {
 588         0xc400, 0xffffffff, 0xfffffffc,
 589         0x802c, 0xffffffff, 0xe0000000,
 590         0x9a60, 0xffffffff, 0x00000100,
 591         0x92a4, 0xffffffff, 0x00000100,
 592         0xc164, 0xffffffff, 0x00000100,
 593         0x9774, 0xffffffff, 0x00000100,
 594         0x8984, 0xffffffff, 0x06000100,
 595         0x8a18, 0xffffffff, 0x00000100,
 596         0x92a0, 0xffffffff, 0x00000100,
 597         0xc380, 0xffffffff, 0x00000100,
 598         0x8b28, 0xffffffff, 0x00000100,
 599         0x9144, 0xffffffff, 0x00000100,
 600         0x8d88, 0xffffffff, 0x00000100,
 601         0x8d8c, 0xffffffff, 0x00000100,
 602         0x9030, 0xffffffff, 0x00000100,
 603         0x9034, 0xffffffff, 0x00000100,
 604         0x9038, 0xffffffff, 0x00000100,
 605         0x903c, 0xffffffff, 0x00000100,
 606         0xad80, 0xffffffff, 0x00000100,
 607         0xac54, 0xffffffff, 0x00000100,
 608         0x897c, 0xffffffff, 0x06000100,
 609         0x9868, 0xffffffff, 0x00000100,
 610         0x9510, 0xffffffff, 0x00000100,
 611         0xaf04, 0xffffffff, 0x00000100,
 612         0xae04, 0xffffffff, 0x00000100,
 613         0x949c, 0xffffffff, 0x00000100,
 614         0x802c, 0xffffffff, 0xe0000000,
 615         0x9160, 0xffffffff, 0x00010000,
 616         0x9164, 0xffffffff, 0x00030002,
 617         0x9168, 0xffffffff, 0x00040007,
 618         0x916c, 0xffffffff, 0x00060005,
 619         0x9170, 0xffffffff, 0x00090008,
 620         0x9174, 0xffffffff, 0x00020001,
 621         0x9178, 0xffffffff, 0x00040003,
 622         0x917c, 0xffffffff, 0x00000007,
 623         0x9180, 0xffffffff, 0x00060005,
 624         0x9184, 0xffffffff, 0x00090008,
 625         0x9188, 0xffffffff, 0x00030002,
 626         0x918c, 0xffffffff, 0x00050004,
 627         0x9190, 0xffffffff, 0x00000008,
 628         0x9194, 0xffffffff, 0x00070006,
 629         0x9198, 0xffffffff, 0x000a0009,
 630         0x919c, 0xffffffff, 0x00040003,
 631         0x91a0, 0xffffffff, 0x00060005,
 632         0x91a4, 0xffffffff, 0x00000009,
 633         0x91a8, 0xffffffff, 0x00080007,
 634         0x91ac, 0xffffffff, 0x000b000a,
 635         0x91b0, 0xffffffff, 0x00050004,
 636         0x91b4, 0xffffffff, 0x00070006,
 637         0x91b8, 0xffffffff, 0x0008000b,
 638         0x91bc, 0xffffffff, 0x000a0009,
 639         0x91c0, 0xffffffff, 0x000d000c,
 640         0x91c4, 0xffffffff, 0x00060005,
 641         0x91c8, 0xffffffff, 0x00080007,
 642         0x91cc, 0xffffffff, 0x0000000b,
 643         0x91d0, 0xffffffff, 0x000a0009,
 644         0x91d4, 0xffffffff, 0x000d000c,
 645         0x91d8, 0xffffffff, 0x00070006,
 646         0x91dc, 0xffffffff, 0x00090008,
 647         0x91e0, 0xffffffff, 0x0000000c,
 648         0x91e4, 0xffffffff, 0x000b000a,
 649         0x91e8, 0xffffffff, 0x000e000d,
 650         0x91ec, 0xffffffff, 0x00080007,
 651         0x91f0, 0xffffffff, 0x000a0009,
 652         0x91f4, 0xffffffff, 0x0000000d,
 653         0x91f8, 0xffffffff, 0x000c000b,
 654         0x91fc, 0xffffffff, 0x000f000e,
 655         0x9200, 0xffffffff, 0x00090008,
 656         0x9204, 0xffffffff, 0x000b000a,
 657         0x9208, 0xffffffff, 0x000c000f,
 658         0x920c, 0xffffffff, 0x000e000d,
 659         0x9210, 0xffffffff, 0x00110010,
 660         0x9214, 0xffffffff, 0x000a0009,
 661         0x9218, 0xffffffff, 0x000c000b,
 662         0x921c, 0xffffffff, 0x0000000f,
 663         0x9220, 0xffffffff, 0x000e000d,
 664         0x9224, 0xffffffff, 0x00110010,
 665         0x9228, 0xffffffff, 0x000b000a,
 666         0x922c, 0xffffffff, 0x000d000c,
 667         0x9230, 0xffffffff, 0x00000010,
 668         0x9234, 0xffffffff, 0x000f000e,
 669         0x9238, 0xffffffff, 0x00120011,
 670         0x923c, 0xffffffff, 0x000c000b,
 671         0x9240, 0xffffffff, 0x000e000d,
 672         0x9244, 0xffffffff, 0x00000011,
 673         0x9248, 0xffffffff, 0x0010000f,
 674         0x924c, 0xffffffff, 0x00130012,
 675         0x9250, 0xffffffff, 0x000d000c,
 676         0x9254, 0xffffffff, 0x000f000e,
 677         0x9258, 0xffffffff, 0x00100013,
 678         0x925c, 0xffffffff, 0x00120011,
 679         0x9260, 0xffffffff, 0x00150014,
 680         0x9264, 0xffffffff, 0x000e000d,
 681         0x9268, 0xffffffff, 0x0010000f,
 682         0x926c, 0xffffffff, 0x00000013,
 683         0x9270, 0xffffffff, 0x00120011,
 684         0x9274, 0xffffffff, 0x00150014,
 685         0x9278, 0xffffffff, 0x000f000e,
 686         0x927c, 0xffffffff, 0x00110010,
 687         0x9280, 0xffffffff, 0x00000014,
 688         0x9284, 0xffffffff, 0x00130012,
 689         0x9288, 0xffffffff, 0x00160015,
 690         0x928c, 0xffffffff, 0x0010000f,
 691         0x9290, 0xffffffff, 0x00120011,
 692         0x9294, 0xffffffff, 0x00000015,
 693         0x9298, 0xffffffff, 0x00140013,
 694         0x929c, 0xffffffff, 0x00170016,
 695         0x9150, 0xffffffff, 0x96940200,
 696         0x8708, 0xffffffff, 0x00900100,
 697         0xc478, 0xffffffff, 0x00000080,
 698         0xc404, 0xffffffff, 0x0020003f,
 699         0x30, 0xffffffff, 0x0000001c,
 700         0x34, 0x000f0000, 0x000f0000,
 701         0x160c, 0xffffffff, 0x00000100,
 702         0x1024, 0xffffffff, 0x00000100,
 703         0x102c, 0x00000101, 0x00000000,
 704         0x20a8, 0xffffffff, 0x00000104,
 705         0x264c, 0x000c0000, 0x000c0000,
 706         0x2648, 0x000c0000, 0x000c0000,
 707         0x55e4, 0xff000fff, 0x00000100,
 708         0x55e8, 0x00000001, 0x00000001,
 709         0x2f50, 0x00000001, 0x00000001,
 710         0x30cc, 0xc0000fff, 0x00000104,
 711         0xc1e4, 0x00000001, 0x00000001,
 712         0xd0c0, 0xfffffff0, 0x00000100,
 713         0xd8c0, 0xfffffff0, 0x00000100
 714 };
 715
 716 static const u32 pitcairn_mgcg_cgcg_init[] =
 717 {
 718         0xc400, 0xffffffff, 0xfffffffc,
 719         0x802c, 0xffffffff, 0xe0000000,
 720         0x9a60, 0xffffffff, 0x00000100,
 721         0x92a4, 0xffffffff, 0x00000100,
 722         0xc164, 0xffffffff, 0x00000100,
 723         0x9774, 0xffffffff, 0x00000100,
 724         0x8984, 0xffffffff, 0x06000100,
 725         0x8a18, 0xffffffff, 0x00000100,
 726         0x92a0, 0xffffffff, 0x00000100,
 727         0xc380, 0xffffffff, 0x00000100,
 728         0x8b28, 0xffffffff, 0x00000100,
 729         0x9144, 0xffffffff, 0x00000100,
 730         0x8d88, 0xffffffff, 0x00000100,
 731         0x8d8c, 0xffffffff, 0x00000100,
 732         0x9030, 0xffffffff, 0x00000100,
 733         0x9034, 0xffffffff, 0x00000100,
 734         0x9038, 0xffffffff, 0x00000100,
 735         0x903c, 0xffffffff, 0x00000100,
 736         0xad80, 0xffffffff, 0x00000100,
 737         0xac54, 0xffffffff, 0x00000100,
 738         0x897c, 0xffffffff, 0x06000100,
 739         0x9868, 0xffffffff, 0x00000100,
 740         0x9510, 0xffffffff, 0x00000100,
 741         0xaf04, 0xffffffff, 0x00000100,
 742         0xae04, 0xffffffff, 0x00000100,
 743         0x949c, 0xffffffff, 0x00000100,
 744         0x802c, 0xffffffff, 0xe0000000,
 745         0x9160, 0xffffffff, 0x00010000,
 746         0x9164, 0xffffffff, 0x00030002,
 747         0x9168, 0xffffffff, 0x00040007,
 748         0x916c, 0xffffffff, 0x00060005,
 749         0x9170, 0xffffffff, 0x00090008,
 750         0x9174, 0xffffffff, 0x00020001,
 751         0x9178, 0xffffffff, 0x00040003,
 752         0x917c, 0xffffffff, 0x00000007,
 753         0x9180, 0xffffffff, 0x00060005,
 754         0x9184, 0xffffffff, 0x00090008,
 755         0x9188, 0xffffffff, 0x00030002,
 756         0x918c, 0xffffffff, 0x00050004,
 757         0x9190, 0xffffffff, 0x00000008,
 758         0x9194, 0xffffffff, 0x00070006,
 759         0x9198, 0xffffffff, 0x000a0009,
 760         0x919c, 0xffffffff, 0x00040003,
 761         0x91a0, 0xffffffff, 0x00060005,
 762         0x91a4, 0xffffffff, 0x00000009,
 763         0x91a8, 0xffffffff, 0x00080007,
 764         0x91ac, 0xffffffff, 0x000b000a,
 765         0x91b0, 0xffffffff, 0x00050004,
 766         0x91b4, 0xffffffff, 0x00070006,
 767         0x91b8, 0xffffffff, 0x0008000b,
 768         0x91bc, 0xffffffff, 0x000a0009,
 769         0x91c0, 0xffffffff, 0x000d000c,
 770         0x9200, 0xffffffff, 0x00090008,
 771         0x9204, 0xffffffff, 0x000b000a,
 772         0x9208, 0xffffffff, 0x000c000f,
 773         0x920c, 0xffffffff, 0x000e000d,
 774         0x9210, 0xffffffff, 0x00110010,
 775         0x9214, 0xffffffff, 0x000a0009,
 776         0x9218, 0xffffffff, 0x000c000b,
 777         0x921c, 0xffffffff, 0x0000000f,
 778         0x9220, 0xffffffff, 0x000e000d,
 779         0x9224, 0xffffffff, 0x00110010,
 780         0x9228, 0xffffffff, 0x000b000a,
 781         0x922c, 0xffffffff, 0x000d000c,
 782         0x9230, 0xffffffff, 0x00000010,
 783         0x9234, 0xffffffff, 0x000f000e,
 784         0x9238, 0xffffffff, 0x00120011,
 785         0x923c, 0xffffffff, 0x000c000b,
 786         0x9240, 0xffffffff, 0x000e000d,
 787         0x9244, 0xffffffff, 0x00000011,
 788         0x9248, 0xffffffff, 0x0010000f,
 789         0x924c, 0xffffffff, 0x00130012,
 790         0x9250, 0xffffffff, 0x000d000c,
 791         0x9254, 0xffffffff, 0x000f000e,
 792         0x9258, 0xffffffff, 0x00100013,
 793         0x925c, 0xffffffff, 0x00120011,
 794         0x9260, 0xffffffff, 0x00150014,
 795         0x9150, 0xffffffff, 0x96940200,
 796         0x8708, 0xffffffff, 0x00900100,
 797         0xc478, 0xffffffff, 0x00000080,
 798         0xc404, 0xffffffff, 0x0020003f,
 799         0x30, 0xffffffff, 0x0000001c,
 800         0x34, 0x000f0000, 0x000f0000,
 801         0x160c, 0xffffffff, 0x00000100,
 802         0x1024, 0xffffffff, 0x00000100,
 803         0x102c, 0x00000101, 0x00000000,
 804         0x20a8, 0xffffffff, 0x00000104,
 805         0x55e4, 0xff000fff, 0x00000100,
 806         0x55e8, 0x00000001, 0x00000001,
 807         0x2f50, 0x00000001, 0x00000001,
 808         0x30cc, 0xc0000fff, 0x00000104,
 809         0xc1e4, 0x00000001, 0x00000001,
 810         0xd0c0, 0xfffffff0, 0x00000100,
 811         0xd8c0, 0xfffffff0, 0x00000100
 812 };
 813
 814 static const u32 verde_mgcg_cgcg_init[] =
 815 {
 816         0xc400, 0xffffffff, 0xfffffffc,
 817         0x802c, 0xffffffff, 0xe0000000,
 818         0x9a60, 0xffffffff, 0x00000100,
 819         0x92a4, 0xffffffff, 0x00000100,
 820         0xc164, 0xffffffff, 0x00000100,
 821         0x9774, 0xffffffff, 0x00000100,
 822         0x8984, 0xffffffff, 0x06000100,
 823         0x8a18, 0xffffffff, 0x00000100,
 824         0x92a0, 0xffffffff, 0x00000100,
 825         0xc380, 0xffffffff, 0x00000100,
 826         0x8b28, 0xffffffff, 0x00000100,
 827         0x9144, 0xffffffff, 0x00000100,
 828         0x8d88, 0xffffffff, 0x00000100,
 829         0x8d8c, 0xffffffff, 0x00000100,
 830         0x9030, 0xffffffff, 0x00000100,
 831         0x9034, 0xffffffff, 0x00000100,
 832         0x9038, 0xffffffff, 0x00000100,
 833         0x903c, 0xffffffff, 0x00000100,
 834         0xad80, 0xffffffff, 0x00000100,
 835         0xac54, 0xffffffff, 0x00000100,
 836         0x897c, 0xffffffff, 0x06000100,
 837         0x9868, 0xffffffff, 0x00000100,
 838         0x9510, 0xffffffff, 0x00000100,
 839         0xaf04, 0xffffffff, 0x00000100,
 840         0xae04, 0xffffffff, 0x00000100,
 841         0x949c, 0xffffffff, 0x00000100,
 842         0x802c, 0xffffffff, 0xe0000000,
 843         0x9160, 0xffffffff, 0x00010000,
 844         0x9164, 0xffffffff, 0x00030002,
 845         0x9168, 0xffffffff, 0x00040007,
 846         0x916c, 0xffffffff, 0x00060005,
 847         0x9170, 0xffffffff, 0x00090008,
 848         0x9174, 0xffffffff, 0x00020001,
 849         0x9178, 0xffffffff, 0x00040003,
 850         0x917c, 0xffffffff, 0x00000007,
 851         0x9180, 0xffffffff, 0x00060005,
 852         0x9184, 0xffffffff, 0x00090008,
 853         0x9188, 0xffffffff, 0x00030002,
 854         0x918c, 0xffffffff, 0x00050004,
 855         0x9190, 0xffffffff, 0x00000008,
 856         0x9194, 0xffffffff, 0x00070006,
 857         0x9198, 0xffffffff, 0x000a0009,
 858         0x919c, 0xffffffff, 0x00040003,
 859         0x91a0, 0xffffffff, 0x00060005,
 860         0x91a4, 0xffffffff, 0x00000009,
 861         0x91a8, 0xffffffff, 0x00080007,
 862         0x91ac, 0xffffffff, 0x000b000a,
 863         0x91b0, 0xffffffff, 0x00050004,
 864         0x91b4, 0xffffffff, 0x00070006,
 865         0x91b8, 0xffffffff, 0x0008000b,
 866         0x91bc, 0xffffffff, 0x000a0009,
 867         0x91c0, 0xffffffff, 0x000d000c,
 868         0x9200, 0xffffffff, 0x00090008,
 869         0x9204, 0xffffffff, 0x000b000a,
 870         0x9208, 0xffffffff, 0x000c000f,
 871         0x920c, 0xffffffff, 0x000e000d,
 872         0x9210, 0xffffffff, 0x00110010,
 873         0x9214, 0xffffffff, 0x000a0009,
 874         0x9218, 0xffffffff, 0x000c000b,
 875         0x921c, 0xffffffff, 0x0000000f,
 876         0x9220, 0xffffffff, 0x000e000d,
 877         0x9224, 0xffffffff, 0x00110010,
 878         0x9228, 0xffffffff, 0x000b000a,
 879         0x922c, 0xffffffff, 0x000d000c,
 880         0x9230, 0xffffffff, 0x00000010,
 881         0x9234, 0xffffffff, 0x000f000e,
 882         0x9238, 0xffffffff, 0x00120011,
 883         0x923c, 0xffffffff, 0x000c000b,
 884         0x9240, 0xffffffff, 0x000e000d,
 885         0x9244, 0xffffffff, 0x00000011,
 886         0x9248, 0xffffffff, 0x0010000f,
 887         0x924c, 0xffffffff, 0x00130012,
 888         0x9250, 0xffffffff, 0x000d000c,
 889         0x9254, 0xffffffff, 0x000f000e,
 890         0x9258, 0xffffffff, 0x00100013,
 891         0x925c, 0xffffffff, 0x00120011,
 892         0x9260, 0xffffffff, 0x00150014,
 893         0x9150, 0xffffffff, 0x96940200,
 894         0x8708, 0xffffffff, 0x00900100,
 895         0xc478, 0xffffffff, 0x00000080,
 896         0xc404, 0xffffffff, 0x0020003f,
 897         0x30, 0xffffffff, 0x0000001c,
 898         0x34, 0x000f0000, 0x000f0000,
 899         0x160c, 0xffffffff, 0x00000100,
 900         0x1024, 0xffffffff, 0x00000100,
 901         0x102c, 0x00000101, 0x00000000,
 902         0x20a8, 0xffffffff, 0x00000104,
 903         0x264c, 0x000c0000, 0x000c0000,
 904         0x2648, 0x000c0000, 0x000c0000,
 905         0x55e4, 0xff000fff, 0x00000100,
 906         0x55e8, 0x00000001, 0x00000001,
 907         0x2f50, 0x00000001, 0x00000001,
 908         0x30cc, 0xc0000fff, 0x00000104,
 909         0xc1e4, 0x00000001, 0x00000001,
 910         0xd0c0, 0xfffffff0, 0x00000100,
 911         0xd8c0, 0xfffffff0, 0x00000100
 912 };
 913
 914 static const u32 oland_mgcg_cgcg_init[] =
 915 {
 916         0xc400, 0xffffffff, 0xfffffffc,
 917         0x802c, 0xffffffff, 0xe0000000,
 918         0x9a60, 0xffffffff, 0x00000100,
 919         0x92a4, 0xffffffff, 0x00000100,
 920         0xc164, 0xffffffff, 0x00000100,
 921         0x9774, 0xffffffff, 0x00000100,
 922         0x8984, 0xffffffff, 0x06000100,
 923         0x8a18, 0xffffffff, 0x00000100,
 924         0x92a0, 0xffffffff, 0x00000100,
 925         0xc380, 0xffffffff, 0x00000100,
 926         0x8b28, 0xffffffff, 0x00000100,
 927         0x9144, 0xffffffff, 0x00000100,
 928         0x8d88, 0xffffffff, 0x00000100,
 929         0x8d8c, 0xffffffff, 0x00000100,
 930         0x9030, 0xffffffff, 0x00000100,
 931         0x9034, 0xffffffff, 0x00000100,
 932         0x9038, 0xffffffff, 0x00000100,
 933         0x903c, 0xffffffff, 0x00000100,
 934         0xad80, 0xffffffff, 0x00000100,
 935         0xac54, 0xffffffff, 0x00000100,
 936         0x897c, 0xffffffff, 0x06000100,
 937         0x9868, 0xffffffff, 0x00000100,
 938         0x9510, 0xffffffff, 0x00000100,
 939         0xaf04, 0xffffffff, 0x00000100,
 940         0xae04, 0xffffffff, 0x00000100,
 941         0x949c, 0xffffffff, 0x00000100,
 942         0x802c, 0xffffffff, 0xe0000000,
 943         0x9160, 0xffffffff, 0x00010000,
 944         0x9164, 0xffffffff, 0x00030002,
 945         0x9168, 0xffffffff, 0x00040007,
 946         0x916c, 0xffffffff, 0x00060005,
 947         0x9170, 0xffffffff, 0x00090008,
 948         0x9174, 0xffffffff, 0x00020001,
 949         0x9178, 0xffffffff, 0x00040003,
 950         0x917c, 0xffffffff, 0x00000007,
 951         0x9180, 0xffffffff, 0x00060005,
 952         0x9184, 0xffffffff, 0x00090008,
 953         0x9188, 0xffffffff, 0x00030002,
 954         0x918c, 0xffffffff, 0x00050004,
 955         0x9190, 0xffffffff, 0x00000008,
 956         0x9194, 0xffffffff, 0x00070006,
 957         0x9198, 0xffffffff, 0x000a0009,
 958         0x919c, 0xffffffff, 0x00040003,
 959         0x91a0, 0xffffffff, 0x00060005,
 960         0x91a4, 0xffffffff, 0x00000009,
 961         0x91a8, 0xffffffff, 0x00080007,
 962         0x91ac, 0xffffffff, 0x000b000a,
 963         0x91b0, 0xffffffff, 0x00050004,
 964         0x91b4, 0xffffffff, 0x00070006,
 965         0x91b8, 0xffffffff, 0x0008000b,
 966         0x91bc, 0xffffffff, 0x000a0009,
 967         0x91c0, 0xffffffff, 0x000d000c,
 968         0x91c4, 0xffffffff, 0x00060005,
 969         0x91c8, 0xffffffff, 0x00080007,
 970         0x91cc, 0xffffffff, 0x0000000b,
 971         0x91d0, 0xffffffff, 0x000a0009,
 972         0x91d4, 0xffffffff, 0x000d000c,
 973         0x9150, 0xffffffff, 0x96940200,
 974         0x8708, 0xffffffff, 0x00900100,
 975         0xc478, 0xffffffff, 0x00000080,
 976         0xc404, 0xffffffff, 0x0020003f,
 977         0x30, 0xffffffff, 0x0000001c,
 978         0x34, 0x000f0000, 0x000f0000,
 979         0x160c, 0xffffffff, 0x00000100,
 980         0x1024, 0xffffffff, 0x00000100,
 981         0x102c, 0x00000101, 0x00000000,
 982         0x20a8, 0xffffffff, 0x00000104,
 983         0x264c, 0x000c0000, 0x000c0000,
 984         0x2648, 0x000c0000, 0x000c0000,
 985         0x55e4, 0xff000fff, 0x00000100,
 986         0x55e8, 0x00000001, 0x00000001,
 987         0x2f50, 0x00000001, 0x00000001,
 988         0x30cc, 0xc0000fff, 0x00000104,
 989         0xc1e4, 0x00000001, 0x00000001,
 990         0xd0c0, 0xfffffff0, 0x00000100,
 991         0xd8c0, 0xfffffff0, 0x00000100
 992 };
 993
 994 static const u32 hainan_mgcg_cgcg_init[] =
 995 {
 996         0xc400, 0xffffffff, 0xfffffffc,
 997         0x802c, 0xffffffff, 0xe0000000,
 998         0x9a60, 0xffffffff, 0x00000100,
 999         0x92a4, 0xffffffff, 0x00000100,
1000         0xc164, 0xffffffff, 0x00000100,
1001         0x9774, 0xffffffff, 0x00000100,
1002         0x8984, 0xffffffff, 0x06000100,
1003         0x8a18, 0xffffffff, 0x00000100,
1004         0x92a0, 0xffffffff, 0x00000100,
1005         0xc380, 0xffffffff, 0x00000100,
1006         0x8b28, 0xffffffff, 0x00000100,
1007         0x9144, 0xffffffff, 0x00000100,
1008         0x8d88, 0xffffffff, 0x00000100,
1009         0x8d8c, 0xffffffff, 0x00000100,
1010         0x9030, 0xffffffff, 0x00000100,
1011         0x9034, 0xffffffff, 0x00000100,
1012         0x9038, 0xffffffff, 0x00000100,
1013         0x903c, 0xffffffff, 0x00000100,
1014         0xad80, 0xffffffff, 0x00000100,
1015         0xac54, 0xffffffff, 0x00000100,
1016         0x897c, 0xffffffff, 0x06000100,
1017         0x9868, 0xffffffff, 0x00000100,
1018         0x9510, 0xffffffff, 0x00000100,
1019         0xaf04, 0xffffffff, 0x00000100,
1020         0xae04, 0xffffffff, 0x00000100,
1021         0x949c, 0xffffffff, 0x00000100,
1022         0x802c, 0xffffffff, 0xe0000000,
1023         0x9160, 0xffffffff, 0x00010000,
1024         0x9164, 0xffffffff, 0x00030002,
1025         0x9168, 0xffffffff, 0x00040007,
1026         0x916c, 0xffffffff, 0x00060005,
1027         0x9170, 0xffffffff, 0x00090008,
1028         0x9174, 0xffffffff, 0x00020001,
1029         0x9178, 0xffffffff, 0x00040003,
1030         0x917c, 0xffffffff, 0x00000007,
1031         0x9180, 0xffffffff, 0x00060005,
1032         0x9184, 0xffffffff, 0x00090008,
1033         0x9188, 0xffffffff, 0x00030002,
1034         0x918c, 0xffffffff, 0x00050004,
1035         0x9190, 0xffffffff, 0x00000008,
1036         0x9194, 0xffffffff, 0x00070006,
1037         0x9198, 0xffffffff, 0x000a0009,
1038         0x919c, 0xffffffff, 0x00040003,
1039         0x91a0, 0xffffffff, 0x00060005,
1040         0x91a4, 0xffffffff, 0x00000009,
1041         0x91a8, 0xffffffff, 0x00080007,
1042         0x91ac, 0xffffffff, 0x000b000a,
1043         0x91b0, 0xffffffff, 0x00050004,
1044         0x91b4, 0xffffffff, 0x00070006,
1045         0x91b8, 0xffffffff, 0x0008000b,
1046         0x91bc, 0xffffffff, 0x000a0009,
1047         0x91c0, 0xffffffff, 0x000d000c,
1048         0x91c4, 0xffffffff, 0x00060005,
1049         0x91c8, 0xffffffff, 0x00080007,
1050         0x91cc, 0xffffffff, 0x0000000b,
1051         0x91d0, 0xffffffff, 0x000a0009,
1052         0x91d4, 0xffffffff, 0x000d000c,
1053         0x9150, 0xffffffff, 0x96940200,
1054         0x8708, 0xffffffff, 0x00900100,
1055         0xc478, 0xffffffff, 0x00000080,
1056         0xc404, 0xffffffff, 0x0020003f,
1057         0x30, 0xffffffff, 0x0000001c,
1058         0x34, 0x000f0000, 0x000f0000,
1059         0x160c, 0xffffffff, 0x00000100,
1060         0x1024, 0xffffffff, 0x00000100,
1061         0x20a8, 0xffffffff, 0x00000104,
1062         0x264c, 0x000c0000, 0x000c0000,
1063         0x2648, 0x000c0000, 0x000c0000,
1064         0x2f50, 0x00000001, 0x00000001,
1065         0x30cc, 0xc0000fff, 0x00000104,
1066         0xc1e4, 0x00000001, 0x00000001,
1067         0xd0c0, 0xfffffff0, 0x00000100,
1068         0xd8c0, 0xfffffff0, 0x00000100
1069 };
1070
1071 static u32 verde_pg_init[] =
1072 {
1073         0x353c, 0xffffffff, 0x40000,
1074         0x3538, 0xffffffff, 0x200010ff,
1075         0x353c, 0xffffffff, 0x0,
1076         0x353c, 0xffffffff, 0x0,
1077         0x353c, 0xffffffff, 0x0,
1078         0x353c, 0xffffffff, 0x0,
1079         0x353c, 0xffffffff, 0x0,
1080         0x353c, 0xffffffff, 0x7007,
1081         0x3538, 0xffffffff, 0x300010ff,
1082         0x353c, 0xffffffff, 0x0,
1083         0x353c, 0xffffffff, 0x0,
1084         0x353c, 0xffffffff, 0x0,
1085         0x353c, 0xffffffff, 0x0,
1086         0x353c, 0xffffffff, 0x0,
1087         0x353c, 0xffffffff, 0x400000,
1088         0x3538, 0xffffffff, 0x100010ff,
1089         0x353c, 0xffffffff, 0x0,
1090         0x353c, 0xffffffff, 0x0,
1091         0x353c, 0xffffffff, 0x0,
1092         0x353c, 0xffffffff, 0x0,
1093         0x353c, 0xffffffff, 0x0,
1094         0x353c, 0xffffffff, 0x120200,
1095         0x3538, 0xffffffff, 0x500010ff,
1096         0x353c, 0xffffffff, 0x0,
1097         0x353c, 0xffffffff, 0x0,
1098         0x353c, 0xffffffff, 0x0,
1099         0x353c, 0xffffffff, 0x0,
1100         0x353c, 0xffffffff, 0x0,
1101         0x353c, 0xffffffff, 0x1e1e16,
1102         0x3538, 0xffffffff, 0x600010ff,
1103         0x353c, 0xffffffff, 0x0,
1104         0x353c, 0xffffffff, 0x0,
1105         0x353c, 0xffffffff, 0x0,
1106         0x353c, 0xffffffff, 0x0,
1107         0x353c, 0xffffffff, 0x0,
1108         0x353c, 0xffffffff, 0x171f1e,
1109         0x3538, 0xffffffff, 0x700010ff,
1110         0x353c, 0xffffffff, 0x0,
1111         0x353c, 0xffffffff, 0x0,
1112         0x353c, 0xffffffff, 0x0,
1113         0x353c, 0xffffffff, 0x0,
1114         0x353c, 0xffffffff, 0x0,
1115         0x353c, 0xffffffff, 0x0,
1116         0x3538, 0xffffffff, 0x9ff,
1117         0x3500, 0xffffffff, 0x0,
1118         0x3504, 0xffffffff, 0x10000800,
1119         0x3504, 0xffffffff, 0xf,
1120         0x3504, 0xffffffff, 0xf,
1121         0x3500, 0xffffffff, 0x4,
1122         0x3504, 0xffffffff, 0x1000051e,
1123         0x3504, 0xffffffff, 0xffff,
1124         0x3504, 0xffffffff, 0xffff,
1125         0x3500, 0xffffffff, 0x8,
1126         0x3504, 0xffffffff, 0x80500,
1127         0x3500, 0xffffffff, 0x12,
1128         0x3504, 0xffffffff, 0x9050c,
1129         0x3500, 0xffffffff, 0x1d,
1130         0x3504, 0xffffffff, 0xb052c,
1131         0x3500, 0xffffffff, 0x2a,
1132         0x3504, 0xffffffff, 0x1053e,
1133         0x3500, 0xffffffff, 0x2d,
1134         0x3504, 0xffffffff, 0x10546,
1135         0x3500, 0xffffffff, 0x30,
1136         0x3504, 0xffffffff, 0xa054e,
1137         0x3500, 0xffffffff, 0x3c,
1138         0x3504, 0xffffffff, 0x1055f,
1139         0x3500, 0xffffffff, 0x3f,
1140         0x3504, 0xffffffff, 0x10567,
1141         0x3500, 0xffffffff, 0x42,
1142         0x3504, 0xffffffff, 0x1056f,
1143         0x3500, 0xffffffff, 0x45,
1144         0x3504, 0xffffffff, 0x10572,
1145         0x3500, 0xffffffff, 0x48,
1146         0x3504, 0xffffffff, 0x20575,
1147         0x3500, 0xffffffff, 0x4c,
1148         0x3504, 0xffffffff, 0x190801,
1149         0x3500, 0xffffffff, 0x67,
1150         0x3504, 0xffffffff, 0x1082a,
1151         0x3500, 0xffffffff, 0x6a,
1152         0x3504, 0xffffffff, 0x1b082d,
1153         0x3500, 0xffffffff, 0x87,
1154         0x3504, 0xffffffff, 0x310851,
1155         0x3500, 0xffffffff, 0xba,
1156         0x3504, 0xffffffff, 0x891,
1157         0x3500, 0xffffffff, 0xbc,
1158         0x3504, 0xffffffff, 0x893,
1159         0x3500, 0xffffffff, 0xbe,
1160         0x3504, 0xffffffff, 0x20895,
1161         0x3500, 0xffffffff, 0xc2,
1162         0x3504, 0xffffffff, 0x20899,
1163         0x3500, 0xffffffff, 0xc6,
1164         0x3504, 0xffffffff, 0x2089d,
1165         0x3500, 0xffffffff, 0xca,
1166         0x3504, 0xffffffff, 0x8a1,
1167         0x3500, 0xffffffff, 0xcc,
1168         0x3504, 0xffffffff, 0x8a3,
1169         0x3500, 0xffffffff, 0xce,
1170         0x3504, 0xffffffff, 0x308a5,
1171         0x3500, 0xffffffff, 0xd3,
1172         0x3504, 0xffffffff, 0x6d08cd,
1173         0x3500, 0xffffffff, 0x142,
1174         0x3504, 0xffffffff, 0x2000095a,
1175         0x3504, 0xffffffff, 0x1,
1176         0x3500, 0xffffffff, 0x144,
1177         0x3504, 0xffffffff, 0x301f095b,
1178         0x3500, 0xffffffff, 0x165,
1179         0x3504, 0xffffffff, 0xc094d,
1180         0x3500, 0xffffffff, 0x173,
1181         0x3504, 0xffffffff, 0xf096d,
1182         0x3500, 0xffffffff, 0x184,
1183         0x3504, 0xffffffff, 0x15097f,
1184         0x3500, 0xffffffff, 0x19b,
1185         0x3504, 0xffffffff, 0xc0998,
1186         0x3500, 0xffffffff, 0x1a9,
1187         0x3504, 0xffffffff, 0x409a7,
1188         0x3500, 0xffffffff, 0x1af,
1189         0x3504, 0xffffffff, 0xcdc,
1190         0x3500, 0xffffffff, 0x1b1,
1191         0x3504, 0xffffffff, 0x800,
1192         0x3508, 0xffffffff, 0x6c9b2000,
1193         0x3510, 0xfc00, 0x2000,
1194         0x3544, 0xffffffff, 0xfc0,
1195         0x28d4, 0x00000100, 0x100
1196 };
1197
1198 static void si_init_golden_registers(struct radeon_device *rdev)
1199 {
1200         switch (rdev->family) {
1201         case CHIP_TAHITI:
1202                 radeon_program_register_sequence(rdev,
1203                                                  tahiti_golden_registers,
1204                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1205                 radeon_program_register_sequence(rdev,
1206                                                  tahiti_golden_rlc_registers,
1207                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1208                 radeon_program_register_sequence(rdev,
1209                                                  tahiti_mgcg_cgcg_init,
1210                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1211                 radeon_program_register_sequence(rdev,
1212                                                  tahiti_golden_registers2,
1213                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1214                 break;
1215         case CHIP_PITCAIRN:
1216                 radeon_program_register_sequence(rdev,
1217                                                  pitcairn_golden_registers,
1218                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1219                 radeon_program_register_sequence(rdev,
1220                                                  pitcairn_golden_rlc_registers,
1221                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1222                 radeon_program_register_sequence(rdev,
1223                                                  pitcairn_mgcg_cgcg_init,
1224                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1225                 break;
1226         case CHIP_VERDE:
1227                 radeon_program_register_sequence(rdev,
1228                                                  verde_golden_registers,
1229                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1230                 radeon_program_register_sequence(rdev,
1231                                                  verde_golden_rlc_registers,
1232                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1233                 radeon_program_register_sequence(rdev,
1234                                                  verde_mgcg_cgcg_init,
1235                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1236                 radeon_program_register_sequence(rdev,
1237                                                  verde_pg_init,
1238                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1239                 break;
1240         case CHIP_OLAND:
1241                 radeon_program_register_sequence(rdev,
1242                                                  oland_golden_registers,
1243                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1244                 radeon_program_register_sequence(rdev,
1245                                                  oland_golden_rlc_registers,
1246                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1247                 radeon_program_register_sequence(rdev,
1248                                                  oland_mgcg_cgcg_init,
1249                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1250                 break;
1251         case CHIP_HAINAN:
1252                 radeon_program_register_sequence(rdev,
1253                                                  hainan_golden_registers,
1254                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1255                 radeon_program_register_sequence(rdev,
1256                                                  hainan_golden_registers2,
1257                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1258                 radeon_program_register_sequence(rdev,
1259                                                  hainan_mgcg_cgcg_init,
1260                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1261                 break;
1262         default:
1263                 break;
1264         }
1265 }
1266
1267 /**
1268  * si_get_allowed_info_register - fetch the register for the info ioctl
1269  *
1270  * @rdev: radeon_device pointer
1271  * @reg: register offset in bytes
1272  * @val: register value
1273  *
1274  * Returns 0 for success or -EINVAL for an invalid register
1275  *
1276  */
1277 int si_get_allowed_info_register(struct radeon_device *rdev,
1278                                  u32 reg, u32 *val)
1279 {
1280         switch (reg) {
1281         case GRBM_STATUS:
1282         case GRBM_STATUS2:
1283         case GRBM_STATUS_SE0:
1284         case GRBM_STATUS_SE1:
1285         case SRBM_STATUS:
1286         case SRBM_STATUS2:
1287         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1288         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1289         case UVD_STATUS:
1290                 *val = RREG32(reg);
1291                 return 0;
1292         default:
1293                 return -EINVAL;
1294         }
1295 }
1296
1297 #define PCIE_BUS_CLK                10000
1298 #define TCLK                        (PCIE_BUS_CLK / 10)
1299
1300 /**
1301  * si_get_xclk - get the xclk
1302  *
1303  * @rdev: radeon_device pointer
1304  *
1305  * Returns the reference clock used by the gfx engine
1306  * (SI).
1307  */
1308 u32 si_get_xclk(struct radeon_device *rdev)
1309 {
1310         u32 reference_clock = rdev->clock.spll.reference_freq;
1311         u32 tmp;
1312
1313         tmp = RREG32(CG_CLKPIN_CNTL_2);
1314         if (tmp & MUX_TCLK_TO_XCLK)
1315                 return TCLK;
1316
1317         tmp = RREG32(CG_CLKPIN_CNTL);
1318         if (tmp & XTALIN_DIVIDE)
1319                 return reference_clock / 4;
1320
1321         return reference_clock;
1322 }
1323
1324 /* get temperature in millidegrees */
1325 int si_get_temp(struct radeon_device *rdev)
1326 {
1327         u32 temp;
1328         int actual_temp = 0;
1329
1330         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1331                 CTF_TEMP_SHIFT;
1332
1333         if (temp & 0x200)
1334                 actual_temp = 255;
1335         else
1336                 actual_temp = temp & 0x1ff;
1337
1338         actual_temp = (actual_temp * 1000);
1339
1340         return actual_temp;
1341 }
1342
1343 #define TAHITI_IO_MC_REGS_SIZE 36
1344
1345 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346         {0x0000006f, 0x03044000},
1347         {0x00000070, 0x0480c018},
1348         {0x00000071, 0x00000040},
1349         {0x00000072, 0x01000000},
1350         {0x00000074, 0x000000ff},
1351         {0x00000075, 0x00143400},
1352         {0x00000076, 0x08ec0800},
1353         {0x00000077, 0x040000cc},
1354         {0x00000079, 0x00000000},
1355         {0x0000007a, 0x21000409},
1356         {0x0000007c, 0x00000000},
1357         {0x0000007d, 0xe8000000},
1358         {0x0000007e, 0x044408a8},
1359         {0x0000007f, 0x00000003},
1360         {0x00000080, 0x00000000},
1361         {0x00000081, 0x01000000},
1362         {0x00000082, 0x02000000},
1363         {0x00000083, 0x00000000},
1364         {0x00000084, 0xe3f3e4f4},
1365         {0x00000085, 0x00052024},
1366         {0x00000087, 0x00000000},
1367         {0x00000088, 0x66036603},
1368         {0x00000089, 0x01000000},
1369         {0x0000008b, 0x1c0a0000},
1370         {0x0000008c, 0xff010000},
1371         {0x0000008e, 0xffffefff},
1372         {0x0000008f, 0xfff3efff},
1373         {0x00000090, 0xfff3efbf},
1374         {0x00000094, 0x00101101},
1375         {0x00000095, 0x00000fff},
1376         {0x00000096, 0x00116fff},
1377         {0x00000097, 0x60010000},
1378         {0x00000098, 0x10010000},
1379         {0x00000099, 0x00006000},
1380         {0x0000009a, 0x00001000},
1381         {0x0000009f, 0x00a77400}
1382 };
1383
1384 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385         {0x0000006f, 0x03044000},
1386         {0x00000070, 0x0480c018},
1387         {0x00000071, 0x00000040},
1388         {0x00000072, 0x01000000},
1389         {0x00000074, 0x000000ff},
1390         {0x00000075, 0x00143400},
1391         {0x00000076, 0x08ec0800},
1392         {0x00000077, 0x040000cc},
1393         {0x00000079, 0x00000000},
1394         {0x0000007a, 0x21000409},
1395         {0x0000007c, 0x00000000},
1396         {0x0000007d, 0xe8000000},
1397         {0x0000007e, 0x044408a8},
1398         {0x0000007f, 0x00000003},
1399         {0x00000080, 0x00000000},
1400         {0x00000081, 0x01000000},
1401         {0x00000082, 0x02000000},
1402         {0x00000083, 0x00000000},
1403         {0x00000084, 0xe3f3e4f4},
1404         {0x00000085, 0x00052024},
1405         {0x00000087, 0x00000000},
1406         {0x00000088, 0x66036603},
1407         {0x00000089, 0x01000000},
1408         {0x0000008b, 0x1c0a0000},
1409         {0x0000008c, 0xff010000},
1410         {0x0000008e, 0xffffefff},
1411         {0x0000008f, 0xfff3efff},
1412         {0x00000090, 0xfff3efbf},
1413         {0x00000094, 0x00101101},
1414         {0x00000095, 0x00000fff},
1415         {0x00000096, 0x00116fff},
1416         {0x00000097, 0x60010000},
1417         {0x00000098, 0x10010000},
1418         {0x00000099, 0x00006000},
1419         {0x0000009a, 0x00001000},
1420         {0x0000009f, 0x00a47400}
1421 };
1422
1423 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424         {0x0000006f, 0x03044000},
1425         {0x00000070, 0x0480c018},
1426         {0x00000071, 0x00000040},
1427         {0x00000072, 0x01000000},
1428         {0x00000074, 0x000000ff},
1429         {0x00000075, 0x00143400},
1430         {0x00000076, 0x08ec0800},
1431         {0x00000077, 0x040000cc},
1432         {0x00000079, 0x00000000},
1433         {0x0000007a, 0x21000409},
1434         {0x0000007c, 0x00000000},
1435         {0x0000007d, 0xe8000000},
1436         {0x0000007e, 0x044408a8},
1437         {0x0000007f, 0x00000003},
1438         {0x00000080, 0x00000000},
1439         {0x00000081, 0x01000000},
1440         {0x00000082, 0x02000000},
1441         {0x00000083, 0x00000000},
1442         {0x00000084, 0xe3f3e4f4},
1443         {0x00000085, 0x00052024},
1444         {0x00000087, 0x00000000},
1445         {0x00000088, 0x66036603},
1446         {0x00000089, 0x01000000},
1447         {0x0000008b, 0x1c0a0000},
1448         {0x0000008c, 0xff010000},
1449         {0x0000008e, 0xffffefff},
1450         {0x0000008f, 0xfff3efff},
1451         {0x00000090, 0xfff3efbf},
1452         {0x00000094, 0x00101101},
1453         {0x00000095, 0x00000fff},
1454         {0x00000096, 0x00116fff},
1455         {0x00000097, 0x60010000},
1456         {0x00000098, 0x10010000},
1457         {0x00000099, 0x00006000},
1458         {0x0000009a, 0x00001000},
1459         {0x0000009f, 0x00a37400}
1460 };
1461
1462 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1463         {0x0000006f, 0x03044000},
1464         {0x00000070, 0x0480c018},
1465         {0x00000071, 0x00000040},
1466         {0x00000072, 0x01000000},
1467         {0x00000074, 0x000000ff},
1468         {0x00000075, 0x00143400},
1469         {0x00000076, 0x08ec0800},
1470         {0x00000077, 0x040000cc},
1471         {0x00000079, 0x00000000},
1472         {0x0000007a, 0x21000409},
1473         {0x0000007c, 0x00000000},
1474         {0x0000007d, 0xe8000000},
1475         {0x0000007e, 0x044408a8},
1476         {0x0000007f, 0x00000003},
1477         {0x00000080, 0x00000000},
1478         {0x00000081, 0x01000000},
1479         {0x00000082, 0x02000000},
1480         {0x00000083, 0x00000000},
1481         {0x00000084, 0xe3f3e4f4},
1482         {0x00000085, 0x00052024},
1483         {0x00000087, 0x00000000},
1484         {0x00000088, 0x66036603},
1485         {0x00000089, 0x01000000},
1486         {0x0000008b, 0x1c0a0000},
1487         {0x0000008c, 0xff010000},
1488         {0x0000008e, 0xffffefff},
1489         {0x0000008f, 0xfff3efff},
1490         {0x00000090, 0xfff3efbf},
1491         {0x00000094, 0x00101101},
1492         {0x00000095, 0x00000fff},
1493         {0x00000096, 0x00116fff},
1494         {0x00000097, 0x60010000},
1495         {0x00000098, 0x10010000},
1496         {0x00000099, 0x00006000},
1497         {0x0000009a, 0x00001000},
1498         {0x0000009f, 0x00a17730}
1499 };
1500
1501 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1502         {0x0000006f, 0x03044000},
1503         {0x00000070, 0x0480c018},
1504         {0x00000071, 0x00000040},
1505         {0x00000072, 0x01000000},
1506         {0x00000074, 0x000000ff},
1507         {0x00000075, 0x00143400},
1508         {0x00000076, 0x08ec0800},
1509         {0x00000077, 0x040000cc},
1510         {0x00000079, 0x00000000},
1511         {0x0000007a, 0x21000409},
1512         {0x0000007c, 0x00000000},
1513         {0x0000007d, 0xe8000000},
1514         {0x0000007e, 0x044408a8},
1515         {0x0000007f, 0x00000003},
1516         {0x00000080, 0x00000000},
1517         {0x00000081, 0x01000000},
1518         {0x00000082, 0x02000000},
1519         {0x00000083, 0x00000000},
1520         {0x00000084, 0xe3f3e4f4},
1521         {0x00000085, 0x00052024},
1522         {0x00000087, 0x00000000},
1523         {0x00000088, 0x66036603},
1524         {0x00000089, 0x01000000},
1525         {0x0000008b, 0x1c0a0000},
1526         {0x0000008c, 0xff010000},
1527         {0x0000008e, 0xffffefff},
1528         {0x0000008f, 0xfff3efff},
1529         {0x00000090, 0xfff3efbf},
1530         {0x00000094, 0x00101101},
1531         {0x00000095, 0x00000fff},
1532         {0x00000096, 0x00116fff},
1533         {0x00000097, 0x60010000},
1534         {0x00000098, 0x10010000},
1535         {0x00000099, 0x00006000},
1536         {0x0000009a, 0x00001000},
1537         {0x0000009f, 0x00a07730}
1538 };
1539
1540 /* ucode loading */
1541 int si_mc_load_microcode(struct radeon_device *rdev)
1542 {
1543         const __be32 *fw_data = NULL;
1544         const __le32 *new_fw_data = NULL;
1545         u32 running, blackout = 0;
1546         u32 *io_mc_regs = NULL;
1547         const __le32 *new_io_mc_regs = NULL;
1548         int i, regs_size, ucode_size;
1549
1550         if (!rdev->mc_fw)
1551                 return -EINVAL;
1552
1553         if (rdev->new_fw) {
1554                 const struct mc_firmware_header_v1_0 *hdr =
1555                         (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1556
1557                 radeon_ucode_print_mc_hdr(&hdr->header);
1558                 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1559                 new_io_mc_regs = (const __le32 *)
1560                         (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1561                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1562                 new_fw_data = (const __le32 *)
1563                         (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1564         } else {
1565                 ucode_size = rdev->mc_fw->size / 4;
1566
1567                 switch (rdev->family) {
1568                 case CHIP_TAHITI:
1569                         io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1570                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1571                         break;
1572                 case CHIP_PITCAIRN:
1573                         io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1574                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1575                         break;
1576                 case CHIP_VERDE:
1577                 default:
1578                         io_mc_regs = (u32 *)&verde_io_mc_regs;
1579                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1580                         break;
1581                 case CHIP_OLAND:
1582                         io_mc_regs = (u32 *)&oland_io_mc_regs;
1583                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1584                         break;
1585                 case CHIP_HAINAN:
1586                         io_mc_regs = (u32 *)&hainan_io_mc_regs;
1587                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1588                         break;
1589                 }
1590                 fw_data = (const __be32 *)rdev->mc_fw->data;
1591         }
1592
1593         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1594
1595         if (running == 0) {
1596                 if (running) {
1597                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1598                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1599                 }
1600
1601                 /* reset the engine and set to writable */
1602                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1603                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1604
1605                 /* load mc io regs */
1606                 for (i = 0; i < regs_size; i++) {
1607                         if (rdev->new_fw) {
1608                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1609                                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1610                         } else {
1611                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1612                                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1613                         }
1614                 }
1615                 /* load the MC ucode */
1616                 for (i = 0; i < ucode_size; i++) {
1617                         if (rdev->new_fw)
1618                                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1619                         else
1620                                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1621                 }
1622
1623                 /* put the engine back into the active state */
1624                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1625                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1626                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1627
1628                 /* wait for training to complete */
1629                 for (i = 0; i < rdev->usec_timeout; i++) {
1630                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1631                                 break;
1632                         udelay(1);
1633                 }
1634                 for (i = 0; i < rdev->usec_timeout; i++) {
1635                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1636                                 break;
1637                         udelay(1);
1638                 }
1639
1640                 if (running)
1641                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1642         }
1643
1644         return 0;
1645 }
1646
1647 static int si_init_microcode(struct radeon_device *rdev)
1648 {
1649         const char *chip_name;
1650         const char *new_chip_name;
1651         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1652         size_t smc_req_size, mc2_req_size;
1653         char fw_name[30];
1654         int err;
1655         int new_fw = 0;
1656
1657         DRM_DEBUG("\n");
1658
1659         switch (rdev->family) {
1660         case CHIP_TAHITI:
1661                 chip_name = "TAHITI";
1662                 new_chip_name = "tahiti";
1663                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1664                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1665                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1666                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1667                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1668                 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1669                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1670                 break;
1671         case CHIP_PITCAIRN:
1672                 chip_name = "PITCAIRN";
1673                 new_chip_name = "pitcairn";
1674                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1675                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1676                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1677                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1678                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1679                 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1680                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1681                 break;
1682         case CHIP_VERDE:
1683                 chip_name = "VERDE";
1684                 new_chip_name = "verde";
1685                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1686                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1687                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1688                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1689                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1690                 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1691                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1692                 break;
1693         case CHIP_OLAND:
1694                 chip_name = "OLAND";
1695                 new_chip_name = "oland";
1696                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1697                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1698                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1699                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1700                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1701                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1702                 break;
1703         case CHIP_HAINAN:
1704                 chip_name = "HAINAN";
1705                 new_chip_name = "hainan";
1706                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1707                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1708                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1709                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1710                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1711                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1712                 break;
1713         default: BUG();
1714         }
1715
1716         DRM_INFO("Loading %s Microcode\n", new_chip_name);
1717
1718         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1719         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1720         if (err) {
1721                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1722                 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1723                 if (err)
1724                         goto out;
1725                 if (rdev->pfp_fw->size != pfp_req_size) {
1726                         printk(KERN_ERR
1727                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1728                                rdev->pfp_fw->size, fw_name);
1729                         err = -EINVAL;
1730                         goto out;
1731                 }
1732         } else {
1733                 err = radeon_ucode_validate(rdev->pfp_fw);
1734                 if (err) {
1735                         printk(KERN_ERR
1736                                "si_cp: validation failed for firmware \"%s\"\n",
1737                                fw_name);
1738                         goto out;
1739                 } else {
1740                         new_fw++;
1741                 }
1742         }
1743
1744         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1745         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1746         if (err) {
1747                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1748                 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1749                 if (err)
1750                         goto out;
1751                 if (rdev->me_fw->size != me_req_size) {
1752                         printk(KERN_ERR
1753                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1754                                rdev->me_fw->size, fw_name);
1755                         err = -EINVAL;
1756                 }
1757         } else {
1758                 err = radeon_ucode_validate(rdev->me_fw);
1759                 if (err) {
1760                         printk(KERN_ERR
1761                                "si_cp: validation failed for firmware \"%s\"\n",
1762                                fw_name);
1763                         goto out;
1764                 } else {
1765                         new_fw++;
1766                 }
1767         }
1768
1769         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1770         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1771         if (err) {
1772                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1773                 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1774                 if (err)
1775                         goto out;
1776                 if (rdev->ce_fw->size != ce_req_size) {
1777                         printk(KERN_ERR
1778                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1779                                rdev->ce_fw->size, fw_name);
1780                         err = -EINVAL;
1781                 }
1782         } else {
1783                 err = radeon_ucode_validate(rdev->ce_fw);
1784                 if (err) {
1785                         printk(KERN_ERR
1786                                "si_cp: validation failed for firmware \"%s\"\n",
1787                                fw_name);
1788                         goto out;
1789                 } else {
1790                         new_fw++;
1791                 }
1792         }
1793
1794         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1795         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1796         if (err) {
1797                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1798                 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1799                 if (err)
1800                         goto out;
1801                 if (rdev->rlc_fw->size != rlc_req_size) {
1802                         printk(KERN_ERR
1803                                "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1804                                rdev->rlc_fw->size, fw_name);
1805                         err = -EINVAL;
1806                 }
1807         } else {
1808                 err = radeon_ucode_validate(rdev->rlc_fw);
1809                 if (err) {
1810                         printk(KERN_ERR
1811                                "si_cp: validation failed for firmware \"%s\"\n",
1812                                fw_name);
1813                         goto out;
1814                 } else {
1815                         new_fw++;
1816                 }
1817         }
1818
1819         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1820         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1821         if (err) {
1822                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1823                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1824                 if (err) {
1825                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1826                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1827                         if (err)
1828                                 goto out;
1829                 }
1830                 if ((rdev->mc_fw->size != mc_req_size) &&
1831                     (rdev->mc_fw->size != mc2_req_size)) {
1832                         printk(KERN_ERR
1833                                "si_mc: Bogus length %zu in firmware \"%s\"\n",
1834                                rdev->mc_fw->size, fw_name);
1835                         err = -EINVAL;
1836                 }
1837                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1838         } else {
1839                 err = radeon_ucode_validate(rdev->mc_fw);
1840                 if (err) {
1841                         printk(KERN_ERR
1842                                "si_cp: validation failed for firmware \"%s\"\n",
1843                                fw_name);
1844                         goto out;
1845                 } else {
1846                         new_fw++;
1847                 }
1848         }
1849
1850         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1851         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1852         if (err) {
1853                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1854                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1855                 if (err) {
1856                         printk(KERN_ERR
1857                                "smc: error loading firmware \"%s\"\n",
1858                                fw_name);
1859                         release_firmware(rdev->smc_fw);
1860                         rdev->smc_fw = NULL;
1861                         err = 0;
1862                 } else if (rdev->smc_fw->size != smc_req_size) {
1863                         printk(KERN_ERR
1864                                "si_smc: Bogus length %zu in firmware \"%s\"\n",
1865                                rdev->smc_fw->size, fw_name);
1866                         err = -EINVAL;
1867                 }
1868         } else {
1869                 err = radeon_ucode_validate(rdev->smc_fw);
1870                 if (err) {
1871                         printk(KERN_ERR
1872                                "si_cp: validation failed for firmware \"%s\"\n",
1873                                fw_name);
1874                         goto out;
1875                 } else {
1876                         new_fw++;
1877                 }
1878         }
1879
1880         if (new_fw == 0) {
1881                 rdev->new_fw = false;
1882         } else if (new_fw < 6) {
1883                 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1884                 err = -EINVAL;
1885         } else {
1886                 rdev->new_fw = true;
1887         }
1888 out:
1889         if (err) {
1890                 if (err != -EINVAL)
1891                         printk(KERN_ERR
1892                                "si_cp: Failed to load firmware \"%s\"\n",
1893                                fw_name);
1894                 release_firmware(rdev->pfp_fw);
1895                 rdev->pfp_fw = NULL;
1896                 release_firmware(rdev->me_fw);
1897                 rdev->me_fw = NULL;
1898                 release_firmware(rdev->ce_fw);
1899                 rdev->ce_fw = NULL;
1900                 release_firmware(rdev->rlc_fw);
1901                 rdev->rlc_fw = NULL;
1902                 release_firmware(rdev->mc_fw);
1903                 rdev->mc_fw = NULL;
1904                 release_firmware(rdev->smc_fw);
1905                 rdev->smc_fw = NULL;
1906         }
1907         return err;
1908 }
1909
1910 /* watermark setup */
1911 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1912                                    struct radeon_crtc *radeon_crtc,
1913                                    struct drm_display_mode *mode,
1914                                    struct drm_display_mode *other_mode)
1915 {
1916         u32 tmp, buffer_alloc, i;
1917         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1918         /*
1919          * Line Buffer Setup
1920          * There are 3 line buffers, each one shared by 2 display controllers.
1921          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1922          * the display controllers.  The paritioning is done via one of four
1923          * preset allocations specified in bits 21:20:
1924          *  0 - half lb
1925          *  2 - whole lb, other crtc must be disabled
1926          */
1927         /* this can get tricky if we have two large displays on a paired group
1928          * of crtcs.  Ideally for multiple large displays we'd assign them to
1929          * non-linked crtcs for maximum line buffer allocation.
1930          */
1931         if (radeon_crtc->base.enabled && mode) {
1932                 if (other_mode) {
1933                         tmp = 0; /* 1/2 */
1934                         buffer_alloc = 1;
1935                 } else {
1936                         tmp = 2; /* whole */
1937                         buffer_alloc = 2;
1938                 }
1939         } else {
1940                 tmp = 0;
1941                 buffer_alloc = 0;
1942         }
1943
1944         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1945                DC_LB_MEMORY_CONFIG(tmp));
1946
1947         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1948                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1949         for (i = 0; i < rdev->usec_timeout; i++) {
1950                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1951                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1952                         break;
1953                 udelay(1);
1954         }
1955
1956         if (radeon_crtc->base.enabled && mode) {
1957                 switch (tmp) {
1958                 case 0:
1959                 default:
1960                         return 4096 * 2;
1961                 case 2:
1962                         return 8192 * 2;
1963                 }
1964         }
1965
1966         /* controller not enabled, so no lb used */
1967         return 0;
1968 }
1969
1970 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1971 {
1972         u32 tmp = RREG32(MC_SHARED_CHMAP);
1973
1974         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1975         case 0:
1976         default:
1977                 return 1;
1978         case 1:
1979                 return 2;
1980         case 2:
1981                 return 4;
1982         case 3:
1983                 return 8;
1984         case 4:
1985                 return 3;
1986         case 5:
1987                 return 6;
1988         case 6:
1989                 return 10;
1990         case 7:
1991                 return 12;
1992         case 8:
1993                 return 16;
1994         }
1995 }
1996
1997 struct dce6_wm_params {
1998         u32 dram_channels; /* number of dram channels */
1999         u32 yclk;          /* bandwidth per dram data pin in kHz */
2000         u32 sclk;          /* engine clock in kHz */
2001         u32 disp_clk;      /* display clock in kHz */
2002         u32 src_width;     /* viewport width */
2003         u32 active_time;   /* active display time in ns */
2004         u32 blank_time;    /* blank time in ns */
2005         bool interlaced;    /* mode is interlaced */
2006         fixed20_12 vsc;    /* vertical scale ratio */
2007         u32 num_heads;     /* number of active crtcs */
2008         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2009         u32 lb_size;       /* line buffer allocated to pipe */
2010         u32 vtaps;         /* vertical scaler taps */
2011 };
2012
2013 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2014 {
2015         /* Calculate raw DRAM Bandwidth */
2016         fixed20_12 dram_efficiency; /* 0.7 */
2017         fixed20_12 yclk, dram_channels, bandwidth;
2018         fixed20_12 a;
2019
2020         a.full = dfixed_const(1000);
2021         yclk.full = dfixed_const(wm->yclk);
2022         yclk.full = dfixed_div(yclk, a);
2023         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2024         a.full = dfixed_const(10);
2025         dram_efficiency.full = dfixed_const(7);
2026         dram_efficiency.full = dfixed_div(dram_efficiency, a);
2027         bandwidth.full = dfixed_mul(dram_channels, yclk);
2028         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2029
2030         return dfixed_trunc(bandwidth);
2031 }
2032
2033 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2034 {
2035         /* Calculate DRAM Bandwidth and the part allocated to display. */
2036         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2037         fixed20_12 yclk, dram_channels, bandwidth;
2038         fixed20_12 a;
2039
2040         a.full = dfixed_const(1000);
2041         yclk.full = dfixed_const(wm->yclk);
2042         yclk.full = dfixed_div(yclk, a);
2043         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2044         a.full = dfixed_const(10);
2045         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2046         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2047         bandwidth.full = dfixed_mul(dram_channels, yclk);
2048         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2049
2050         return dfixed_trunc(bandwidth);
2051 }
2052
2053 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2054 {
2055         /* Calculate the display Data return Bandwidth */
2056         fixed20_12 return_efficiency; /* 0.8 */
2057         fixed20_12 sclk, bandwidth;
2058         fixed20_12 a;
2059
2060         a.full = dfixed_const(1000);
2061         sclk.full = dfixed_const(wm->sclk);
2062         sclk.full = dfixed_div(sclk, a);
2063         a.full = dfixed_const(10);
2064         return_efficiency.full = dfixed_const(8);
2065         return_efficiency.full = dfixed_div(return_efficiency, a);
2066         a.full = dfixed_const(32);
2067         bandwidth.full = dfixed_mul(a, sclk);
2068         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2069
2070         return dfixed_trunc(bandwidth);
2071 }
2072
2073 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2074 {
2075         return 32;
2076 }
2077
2078 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2079 {
2080         /* Calculate the DMIF Request Bandwidth */
2081         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2082         fixed20_12 disp_clk, sclk, bandwidth;
2083         fixed20_12 a, b1, b2;
2084         u32 min_bandwidth;
2085
2086         a.full = dfixed_const(1000);
2087         disp_clk.full = dfixed_const(wm->disp_clk);
2088         disp_clk.full = dfixed_div(disp_clk, a);
2089         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2090         b1.full = dfixed_mul(a, disp_clk);
2091
2092         a.full = dfixed_const(1000);
2093         sclk.full = dfixed_const(wm->sclk);
2094         sclk.full = dfixed_div(sclk, a);
2095         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2096         b2.full = dfixed_mul(a, sclk);
2097
2098         a.full = dfixed_const(10);
2099         disp_clk_request_efficiency.full = dfixed_const(8);
2100         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2101
2102         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2103
2104         a.full = dfixed_const(min_bandwidth);
2105         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2106
2107         return dfixed_trunc(bandwidth);
2108 }
2109
2110 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2111 {
2112         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2113         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2114         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2115         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2116
2117         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2118 }
2119
2120 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2121 {
2122         /* Calculate the display mode Average Bandwidth
2123          * DisplayMode should contain the source and destination dimensions,
2124          * timing, etc.
2125          */
2126         fixed20_12 bpp;
2127         fixed20_12 line_time;
2128         fixed20_12 src_width;
2129         fixed20_12 bandwidth;
2130         fixed20_12 a;
2131
2132         a.full = dfixed_const(1000);
2133         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2134         line_time.full = dfixed_div(line_time, a);
2135         bpp.full = dfixed_const(wm->bytes_per_pixel);
2136         src_width.full = dfixed_const(wm->src_width);
2137         bandwidth.full = dfixed_mul(src_width, bpp);
2138         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2139         bandwidth.full = dfixed_div(bandwidth, line_time);
2140
2141         return dfixed_trunc(bandwidth);
2142 }
2143
2144 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2145 {
2146         /* First calcualte the latency in ns */
2147         u32 mc_latency = 2000; /* 2000 ns. */
2148         u32 available_bandwidth = dce6_available_bandwidth(wm);
2149         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2150         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2151         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2152         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2153                 (wm->num_heads * cursor_line_pair_return_time);
2154         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2155         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2156         u32 tmp, dmif_size = 12288;
2157         fixed20_12 a, b, c;
2158
2159         if (wm->num_heads == 0)
2160                 return 0;
2161
2162         a.full = dfixed_const(2);
2163         b.full = dfixed_const(1);
2164         if ((wm->vsc.full > a.full) ||
2165             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2166             (wm->vtaps >= 5) ||
2167             ((wm->vsc.full >= a.full) && wm->interlaced))
2168                 max_src_lines_per_dst_line = 4;
2169         else
2170                 max_src_lines_per_dst_line = 2;
2171
2172         a.full = dfixed_const(available_bandwidth);
2173         b.full = dfixed_const(wm->num_heads);
2174         a.full = dfixed_div(a, b);
2175
2176         b.full = dfixed_const(mc_latency + 512);
2177         c.full = dfixed_const(wm->disp_clk);
2178         b.full = dfixed_div(b, c);
2179
2180         c.full = dfixed_const(dmif_size);
2181         b.full = dfixed_div(c, b);
2182
2183         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2184
2185         b.full = dfixed_const(1000);
2186         c.full = dfixed_const(wm->disp_clk);
2187         b.full = dfixed_div(c, b);
2188         c.full = dfixed_const(wm->bytes_per_pixel);
2189         b.full = dfixed_mul(b, c);
2190
2191         lb_fill_bw = min(tmp, dfixed_trunc(b));
2192
2193         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2194         b.full = dfixed_const(1000);
2195         c.full = dfixed_const(lb_fill_bw);
2196         b.full = dfixed_div(c, b);
2197         a.full = dfixed_div(a, b);
2198         line_fill_time = dfixed_trunc(a);
2199
2200         if (line_fill_time < wm->active_time)
2201                 return latency;
2202         else
2203                 return latency + (line_fill_time - wm->active_time);
2204
2205 }
2206
2207 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2208 {
2209         if (dce6_average_bandwidth(wm) <=
2210             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2211                 return true;
2212         else
2213                 return false;
2214 };
2215
2216 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2217 {
2218         if (dce6_average_bandwidth(wm) <=
2219             (dce6_available_bandwidth(wm) / wm->num_heads))
2220                 return true;
2221         else
2222                 return false;
2223 };
2224
2225 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2226 {
2227         u32 lb_partitions = wm->lb_size / wm->src_width;
2228         u32 line_time = wm->active_time + wm->blank_time;
2229         u32 latency_tolerant_lines;
2230         u32 latency_hiding;
2231         fixed20_12 a;
2232
2233         a.full = dfixed_const(1);
2234         if (wm->vsc.full > a.full)
2235                 latency_tolerant_lines = 1;
2236         else {
2237                 if (lb_partitions <= (wm->vtaps + 1))
2238                         latency_tolerant_lines = 1;
2239                 else
2240                         latency_tolerant_lines = 2;
2241         }
2242
2243         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2244
2245         if (dce6_latency_watermark(wm) <= latency_hiding)
2246                 return true;
2247         else
2248                 return false;
2249 }
2250
2251 static void dce6_program_watermarks(struct radeon_device *rdev,
2252                                          struct radeon_crtc *radeon_crtc,
2253                                          u32 lb_size, u32 num_heads)
2254 {
2255         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2256         struct dce6_wm_params wm_low, wm_high;
2257         u32 dram_channels;
2258         u32 pixel_period;
2259         u32 line_time = 0;
2260         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2261         u32 priority_a_mark = 0, priority_b_mark = 0;
2262         u32 priority_a_cnt = PRIORITY_OFF;
2263         u32 priority_b_cnt = PRIORITY_OFF;
2264         u32 tmp, arb_control3;
2265         fixed20_12 a, b, c;
2266
2267         if (radeon_crtc->base.enabled && num_heads && mode) {
2268                 pixel_period = 1000000 / (u32)mode->clock;
2269                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2270                 priority_a_cnt = 0;
2271                 priority_b_cnt = 0;
2272
2273                 if (rdev->family == CHIP_ARUBA)
2274                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275                 else
2276                         dram_channels = si_get_number_of_dram_channels(rdev);
2277
2278                 /* watermark for high clocks */
2279                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2280                         wm_high.yclk =
2281                                 radeon_dpm_get_mclk(rdev, false) * 10;
2282                         wm_high.sclk =
2283                                 radeon_dpm_get_sclk(rdev, false) * 10;
2284                 } else {
2285                         wm_high.yclk = rdev->pm.current_mclk * 10;
2286                         wm_high.sclk = rdev->pm.current_sclk * 10;
2287                 }
2288
2289                 wm_high.disp_clk = mode->clock;
2290                 wm_high.src_width = mode->crtc_hdisplay;
2291                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2292                 wm_high.blank_time = line_time - wm_high.active_time;
2293                 wm_high.interlaced = false;
2294                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2295                         wm_high.interlaced = true;
2296                 wm_high.vsc = radeon_crtc->vsc;
2297                 wm_high.vtaps = 1;
2298                 if (radeon_crtc->rmx_type != RMX_OFF)
2299                         wm_high.vtaps = 2;
2300                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2301                 wm_high.lb_size = lb_size;
2302                 wm_high.dram_channels = dram_channels;
2303                 wm_high.num_heads = num_heads;
2304
2305                 /* watermark for low clocks */
2306                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2307                         wm_low.yclk =
2308                                 radeon_dpm_get_mclk(rdev, true) * 10;
2309                         wm_low.sclk =
2310                                 radeon_dpm_get_sclk(rdev, true) * 10;
2311                 } else {
2312                         wm_low.yclk = rdev->pm.current_mclk * 10;
2313                         wm_low.sclk = rdev->pm.current_sclk * 10;
2314                 }
2315
2316                 wm_low.disp_clk = mode->clock;
2317                 wm_low.src_width = mode->crtc_hdisplay;
2318                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2319                 wm_low.blank_time = line_time - wm_low.active_time;
2320                 wm_low.interlaced = false;
2321                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2322                         wm_low.interlaced = true;
2323                 wm_low.vsc = radeon_crtc->vsc;
2324                 wm_low.vtaps = 1;
2325                 if (radeon_crtc->rmx_type != RMX_OFF)
2326                         wm_low.vtaps = 2;
2327                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2328                 wm_low.lb_size = lb_size;
2329                 wm_low.dram_channels = dram_channels;
2330                 wm_low.num_heads = num_heads;
2331
2332                 /* set for high clocks */
2333                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2334                 /* set for low clocks */
2335                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2336
2337                 /* possibly force display priority to high */
2338                 /* should really do this at mode validation time... */
2339                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2340                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2341                     !dce6_check_latency_hiding(&wm_high) ||
2342                     (rdev->disp_priority == 2)) {
2343                         DRM_DEBUG_KMS("force priority to high\n");
2344                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2345                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2346                 }
2347                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2348                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2349                     !dce6_check_latency_hiding(&wm_low) ||
2350                     (rdev->disp_priority == 2)) {
2351                         DRM_DEBUG_KMS("force priority to high\n");
2352                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2353                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2354                 }
2355
2356                 a.full = dfixed_const(1000);
2357                 b.full = dfixed_const(mode->clock);
2358                 b.full = dfixed_div(b, a);
2359                 c.full = dfixed_const(latency_watermark_a);
2360                 c.full = dfixed_mul(c, b);
2361                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2362                 c.full = dfixed_div(c, a);
2363                 a.full = dfixed_const(16);
2364                 c.full = dfixed_div(c, a);
2365                 priority_a_mark = dfixed_trunc(c);
2366                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2367
2368                 a.full = dfixed_const(1000);
2369                 b.full = dfixed_const(mode->clock);
2370                 b.full = dfixed_div(b, a);
2371                 c.full = dfixed_const(latency_watermark_b);
2372                 c.full = dfixed_mul(c, b);
2373                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2374                 c.full = dfixed_div(c, a);
2375                 a.full = dfixed_const(16);
2376                 c.full = dfixed_div(c, a);
2377                 priority_b_mark = dfixed_trunc(c);
2378                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2379
2380                 /* Save number of lines the linebuffer leads before the scanout */
2381                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2382         }
2383
2384         /* select wm A */
2385         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2386         tmp = arb_control3;
2387         tmp &= ~LATENCY_WATERMARK_MASK(3);
2388         tmp |= LATENCY_WATERMARK_MASK(1);
2389         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2390         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2391                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2392                 LATENCY_HIGH_WATERMARK(line_time)));
2393         /* select wm B */
2394         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2395         tmp &= ~LATENCY_WATERMARK_MASK(3);
2396         tmp |= LATENCY_WATERMARK_MASK(2);
2397         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2398         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2399                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2400                 LATENCY_HIGH_WATERMARK(line_time)));
2401         /* restore original selection */
2402         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2403
2404         /* write the priority marks */
2405         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2406         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2407
2408         /* save values for DPM */
2409         radeon_crtc->line_time = line_time;
2410         radeon_crtc->wm_high = latency_watermark_a;
2411         radeon_crtc->wm_low = latency_watermark_b;
2412 }
2413
2414 void dce6_bandwidth_update(struct radeon_device *rdev)
2415 {
2416         struct drm_display_mode *mode0 = NULL;
2417         struct drm_display_mode *mode1 = NULL;
2418         u32 num_heads = 0, lb_size;
2419         int i;
2420
2421         if (!rdev->mode_info.mode_config_initialized)
2422                 return;
2423
2424         radeon_update_display_priority(rdev);
2425
2426         for (i = 0; i < rdev->num_crtc; i++) {
2427                 if (rdev->mode_info.crtcs[i]->base.enabled)
2428                         num_heads++;
2429         }
2430         for (i = 0; i < rdev->num_crtc; i += 2) {
2431                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2432                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2433                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2434                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2435                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2436                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2437         }
2438 }
2439
2440 /*
2441  * Core functions
2442  */
2443 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2444 {
2445         const u32 num_tile_mode_states = 32;
2446         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2447
2448         switch (rdev->config.si.mem_row_size_in_kb) {
2449         case 1:
2450                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2451                 break;
2452         case 2:
2453         default:
2454                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2455                 break;
2456         case 4:
2457                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2458                 break;
2459         }
2460
2461         if ((rdev->family == CHIP_TAHITI) ||
2462             (rdev->family == CHIP_PITCAIRN)) {
2463                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2464                         switch (reg_offset) {
2465                         case 0:  /* non-AA compressed depth or any compressed stencil */
2466                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2467                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2468                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2469                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2470                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2471                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2472                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2473                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2474                                 break;
2475                         case 1:  /* 2xAA/4xAA compressed depth only */
2476                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2477                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2478                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2479                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2480                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2481                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2483                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2484                                 break;
2485                         case 2:  /* 8xAA compressed depth only */
2486                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2487                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2488                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2489                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2490                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2491                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2493                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2494                                 break;
2495                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2496                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2500                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2501                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2504                                 break;
2505                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2506                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2507                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2509                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2511                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2513                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2514                                 break;
2515                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2516                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2518                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2519                                                  TILE_SPLIT(split_equal_to_row_size) |
2520                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2521                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2523                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2524                                 break;
2525                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2526                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2528                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2529                                                  TILE_SPLIT(split_equal_to_row_size) |
2530                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2531                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2532                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2533                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2534                                 break;
2535                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2536                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2538                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2539                                                  TILE_SPLIT(split_equal_to_row_size) |
2540                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2541                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2542                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2543                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2544                                 break;
2545                         case 8:  /* 1D and 1D Array Surfaces */
2546                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2547                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2548                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2549                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2550                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2551                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2554                                 break;
2555                         case 9:  /* Displayable maps. */
2556                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2557                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2558                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2559                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2560                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2561                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2562                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2563                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2564                                 break;
2565                         case 10:  /* Display 8bpp. */
2566                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2568                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2569                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2570                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2571                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2572                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2573                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2574                                 break;
2575                         case 11:  /* Display 16bpp. */
2576                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2578                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2579                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2580                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2581                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2583                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2584                                 break;
2585                         case 12:  /* Display 32bpp. */
2586                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2590                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2591                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2594                                 break;
2595                         case 13:  /* Thin. */
2596                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2598                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2601                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604                                 break;
2605                         case 14:  /* Thin 8 bpp. */
2606                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2607                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2608                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2609                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2610                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2611                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2614                                 break;
2615                         case 15:  /* Thin 16 bpp. */
2616                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2618                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2619                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2620                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2621                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2622                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2623                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2624                                 break;
2625                         case 16:  /* Thin 32 bpp. */
2626                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2627                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2628                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2629                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2630                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2631                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2634                                 break;
2635                         case 17:  /* Thin 64 bpp. */
2636                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2637                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2638                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2639                                                  TILE_SPLIT(split_equal_to_row_size) |
2640                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2641                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2643                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2644                                 break;
2645                         case 21:  /* 8 bpp PRT. */
2646                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2647                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2648                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2649                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2650                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2651                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2652                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2653                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2654                                 break;
2655                         case 22:  /* 16 bpp PRT */
2656                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2658                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2659                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2660                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2661                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2662                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2663                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2664                                 break;
2665                         case 23:  /* 32 bpp PRT */
2666                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2668                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2669                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2670                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2671                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2673                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2674                                 break;
2675                         case 24:  /* 64 bpp PRT */
2676                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2680                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2681                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2683                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684                                 break;
2685                         case 25:  /* 128 bpp PRT */
2686                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2690                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2691                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2692                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2693                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2694                                 break;
2695                         default:
2696                                 gb_tile_moden = 0;
2697                                 break;
2698                         }
2699                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2700                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2701                 }
2702         } else if ((rdev->family == CHIP_VERDE) ||
2703                    (rdev->family == CHIP_OLAND) ||
2704                    (rdev->family == CHIP_HAINAN)) {
2705                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2706                         switch (reg_offset) {
2707                         case 0:  /* non-AA compressed depth or any compressed stencil */
2708                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2709                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2710                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2711                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2712                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2713                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2715                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2716                                 break;
2717                         case 1:  /* 2xAA/4xAA compressed depth only */
2718                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2719                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2720                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2721                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2722                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2723                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2725                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2726                                 break;
2727                         case 2:  /* 8xAA compressed depth only */
2728                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2730                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2731                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2732                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2733                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2734                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2735                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2736                                 break;
2737                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2738                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2740                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2742                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2743                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746                                 break;
2747                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2748                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2749                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2750                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2752                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2753                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2755                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2756                                 break;
2757                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2758                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2759                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2760                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2761                                                  TILE_SPLIT(split_equal_to_row_size) |
2762                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2763                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2764                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2765                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2766                                 break;
2767                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2768                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2770                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771                                                  TILE_SPLIT(split_equal_to_row_size) |
2772                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2773                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2774                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2775                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2776                                 break;
2777                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2778                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2779                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2780                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2781                                                  TILE_SPLIT(split_equal_to_row_size) |
2782                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2783                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2785                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2786                                 break;
2787                         case 8:  /* 1D and 1D Array Surfaces */
2788                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2789                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2790                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2791                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2792                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2793                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2794                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2795                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2796                                 break;
2797                         case 9:  /* Displayable maps. */
2798                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2799                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2800                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2801                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2802                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2803                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2804                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2805                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2806                                 break;
2807                         case 10:  /* Display 8bpp. */
2808                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2810                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2811                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2812                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2813                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2814                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2815                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2816                                 break;
2817                         case 11:  /* Display 16bpp. */
2818                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2819                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2820                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2821                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2822                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2823                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2824                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2825                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2826                                 break;
2827                         case 12:  /* Display 32bpp. */
2828                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2830                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2831                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2832                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2833                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2835                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2836                                 break;
2837                         case 13:  /* Thin. */
2838                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2839                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2840                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2841                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2842                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2843                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2845                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846                                 break;
2847                         case 14:  /* Thin 8 bpp. */
2848                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2849                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2850                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2851                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2852                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2853                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2854                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2855                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2856                                 break;
2857                         case 15:  /* Thin 16 bpp. */
2858                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2859                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2860                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2861                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2862                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2863                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2864                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2865                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2866                                 break;
2867                         case 16:  /* Thin 32 bpp. */
2868                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2869                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2870                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2871                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2872                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2873                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2874                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2875                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2876                                 break;
2877                         case 17:  /* Thin 64 bpp. */
2878                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2879                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2880                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2881                                                  TILE_SPLIT(split_equal_to_row_size) |
2882                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2883                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2884                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2885                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2886                                 break;
2887                         case 21:  /* 8 bpp PRT. */
2888                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2890                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2891                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2892                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2893                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2894                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2895                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2896                                 break;
2897                         case 22:  /* 16 bpp PRT */
2898                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2899                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2900                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2901                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2902                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2903                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2904                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2905                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2906                                 break;
2907                         case 23:  /* 32 bpp PRT */
2908                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2909                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2910                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2911                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2912                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2913                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2914                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2915                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2916                                 break;
2917                         case 24:  /* 64 bpp PRT */
2918                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2919                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2920                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2921                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2922                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2923                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2924                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2925                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2926                                 break;
2927                         case 25:  /* 128 bpp PRT */
2928                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2929                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2930                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2931                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2932                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2933                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2934                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2935                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2936                                 break;
2937                         default:
2938                                 gb_tile_moden = 0;
2939                                 break;
2940                         }
2941                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2942                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2943                 }
2944         } else
2945                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2946 }
2947
2948 static void si_select_se_sh(struct radeon_device *rdev,
2949                             u32 se_num, u32 sh_num)
2950 {
2951         u32 data = INSTANCE_BROADCAST_WRITES;
2952
2953         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2954                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2955         else if (se_num == 0xffffffff)
2956                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2957         else if (sh_num == 0xffffffff)
2958                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2959         else
2960                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2961         WREG32(GRBM_GFX_INDEX, data);
2962 }
2963
2964 static u32 si_create_bitmask(u32 bit_width)
2965 {
2966         u32 i, mask = 0;
2967
2968         for (i = 0; i < bit_width; i++) {
2969                 mask <<= 1;
2970                 mask |= 1;
2971         }
2972         return mask;
2973 }
2974
2975 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2976 {
2977         u32 data, mask;
2978
2979         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2980         if (data & 1)
2981                 data &= INACTIVE_CUS_MASK;
2982         else
2983                 data = 0;
2984         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2985
2986         data >>= INACTIVE_CUS_SHIFT;
2987
2988         mask = si_create_bitmask(cu_per_sh);
2989
2990         return ~data & mask;
2991 }
2992
2993 static void si_setup_spi(struct radeon_device *rdev,
2994                          u32 se_num, u32 sh_per_se,
2995                          u32 cu_per_sh)
2996 {
2997         int i, j, k;
2998         u32 data, mask, active_cu;
2999
3000         for (i = 0; i < se_num; i++) {
3001                 for (j = 0; j < sh_per_se; j++) {
3002                         si_select_se_sh(rdev, i, j);
3003                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3004                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3005
3006                         mask = 1;
3007                         for (k = 0; k < 16; k++) {
3008                                 mask <<= k;
3009                                 if (active_cu & mask) {
3010                                         data &= ~mask;
3011                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3012                                         break;
3013                                 }
3014                         }
3015                 }
3016         }
3017         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3018 }
3019
3020 static u32 si_get_rb_disabled(struct radeon_device *rdev,
3021                               u32 max_rb_num_per_se,
3022                               u32 sh_per_se)
3023 {
3024         u32 data, mask;
3025
3026         data = RREG32(CC_RB_BACKEND_DISABLE);
3027         if (data & 1)
3028                 data &= BACKEND_DISABLE_MASK;
3029         else
3030                 data = 0;
3031         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3032
3033         data >>= BACKEND_DISABLE_SHIFT;
3034
3035         mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3036
3037         return data & mask;
3038 }
3039
3040 static void si_setup_rb(struct radeon_device *rdev,
3041                         u32 se_num, u32 sh_per_se,
3042                         u32 max_rb_num_per_se)
3043 {
3044         int i, j;
3045         u32 data, mask;
3046         u32 disabled_rbs = 0;
3047         u32 enabled_rbs = 0;
3048
3049         for (i = 0; i < se_num; i++) {
3050                 for (j = 0; j < sh_per_se; j++) {
3051                         si_select_se_sh(rdev, i, j);
3052                         data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3053                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3054                 }
3055         }
3056         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3057
3058         mask = 1;
3059         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3060                 if (!(disabled_rbs & mask))
3061                         enabled_rbs |= mask;
3062                 mask <<= 1;
3063         }
3064
3065         rdev->config.si.backend_enable_mask = enabled_rbs;
3066
3067         for (i = 0; i < se_num; i++) {
3068                 si_select_se_sh(rdev, i, 0xffffffff);
3069                 data = 0;
3070                 for (j = 0; j < sh_per_se; j++) {
3071                         switch (enabled_rbs & 3) {
3072                         case 1:
3073                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3074                                 break;
3075                         case 2:
3076                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3077                                 break;
3078                         case 3:
3079                         default:
3080                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3081                                 break;
3082                         }
3083                         enabled_rbs >>= 2;
3084                 }
3085                 WREG32(PA_SC_RASTER_CONFIG, data);
3086         }
3087         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3088 }
3089
3090 static void si_gpu_init(struct radeon_device *rdev)
3091 {
3092         u32 gb_addr_config = 0;
3093         u32 mc_shared_chmap, mc_arb_ramcfg;
3094         u32 sx_debug_1;
3095         u32 hdp_host_path_cntl;
3096         u32 tmp;
3097         int i, j;
3098
3099         switch (rdev->family) {
3100         case CHIP_TAHITI:
3101                 rdev->config.si.max_shader_engines = 2;
3102                 rdev->config.si.max_tile_pipes = 12;
3103                 rdev->config.si.max_cu_per_sh = 8;
3104                 rdev->config.si.max_sh_per_se = 2;
3105                 rdev->config.si.max_backends_per_se = 4;
3106                 rdev->config.si.max_texture_channel_caches = 12;
3107                 rdev->config.si.max_gprs = 256;
3108                 rdev->config.si.max_gs_threads = 32;
3109                 rdev->config.si.max_hw_contexts = 8;
3110
3111                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3112                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3113                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3114                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3115                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3116                 break;
3117         case CHIP_PITCAIRN:
3118                 rdev->config.si.max_shader_engines = 2;
3119                 rdev->config.si.max_tile_pipes = 8;
3120                 rdev->config.si.max_cu_per_sh = 5;
3121                 rdev->config.si.max_sh_per_se = 2;
3122                 rdev->config.si.max_backends_per_se = 4;
3123                 rdev->config.si.max_texture_channel_caches = 8;
3124                 rdev->config.si.max_gprs = 256;
3125                 rdev->config.si.max_gs_threads = 32;
3126                 rdev->config.si.max_hw_contexts = 8;
3127
3128                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3129                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3130                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3131                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3132                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3133                 break;
3134         case CHIP_VERDE:
3135         default:
3136                 rdev->config.si.max_shader_engines = 1;
3137                 rdev->config.si.max_tile_pipes = 4;
3138                 rdev->config.si.max_cu_per_sh = 5;
3139                 rdev->config.si.max_sh_per_se = 2;
3140                 rdev->config.si.max_backends_per_se = 4;
3141                 rdev->config.si.max_texture_channel_caches = 4;
3142                 rdev->config.si.max_gprs = 256;
3143                 rdev->config.si.max_gs_threads = 32;
3144                 rdev->config.si.max_hw_contexts = 8;
3145
3146                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3147                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3148                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3149                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3150                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3151                 break;
3152         case CHIP_OLAND:
3153                 rdev->config.si.max_shader_engines = 1;
3154                 rdev->config.si.max_tile_pipes = 4;
3155                 rdev->config.si.max_cu_per_sh = 6;
3156                 rdev->config.si.max_sh_per_se = 1;
3157                 rdev->config.si.max_backends_per_se = 2;
3158                 rdev->config.si.max_texture_channel_caches = 4;
3159                 rdev->config.si.max_gprs = 256;
3160                 rdev->config.si.max_gs_threads = 16;
3161                 rdev->config.si.max_hw_contexts = 8;
3162
3163                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3164                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3165                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3166                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3167                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3168                 break;
3169         case CHIP_HAINAN:
3170                 rdev->config.si.max_shader_engines = 1;
3171                 rdev->config.si.max_tile_pipes = 4;
3172                 rdev->config.si.max_cu_per_sh = 5;
3173                 rdev->config.si.max_sh_per_se = 1;
3174                 rdev->config.si.max_backends_per_se = 1;
3175                 rdev->config.si.max_texture_channel_caches = 2;
3176                 rdev->config.si.max_gprs = 256;
3177                 rdev->config.si.max_gs_threads = 16;
3178                 rdev->config.si.max_hw_contexts = 8;
3179
3180                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3181                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3182                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3183                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3184                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3185                 break;
3186         }
3187
3188         /* Initialize HDP */
3189         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3190                 WREG32((0x2c14 + j), 0x00000000);
3191                 WREG32((0x2c18 + j), 0x00000000);
3192                 WREG32((0x2c1c + j), 0x00000000);
3193                 WREG32((0x2c20 + j), 0x00000000);
3194                 WREG32((0x2c24 + j), 0x00000000);
3195         }
3196
3197         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3198         WREG32(SRBM_INT_CNTL, 1);
3199         WREG32(SRBM_INT_ACK, 1);
3200
3201         evergreen_fix_pci_max_read_req_size(rdev);
3202
3203         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3204
3205         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3206         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3207
3208         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3209         rdev->config.si.mem_max_burst_length_bytes = 256;
3210         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3211         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3212         if (rdev->config.si.mem_row_size_in_kb > 4)
3213                 rdev->config.si.mem_row_size_in_kb = 4;
3214         /* XXX use MC settings? */
3215         rdev->config.si.shader_engine_tile_size = 32;
3216         rdev->config.si.num_gpus = 1;
3217         rdev->config.si.multi_gpu_tile_size = 64;
3218
3219         /* fix up row size */
3220         gb_addr_config &= ~ROW_SIZE_MASK;
3221         switch (rdev->config.si.mem_row_size_in_kb) {
3222         case 1:
3223         default:
3224                 gb_addr_config |= ROW_SIZE(0);
3225                 break;
3226         case 2:
3227                 gb_addr_config |= ROW_SIZE(1);
3228                 break;
3229         case 4:
3230                 gb_addr_config |= ROW_SIZE(2);
3231                 break;
3232         }
3233
3234         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3235          * not have bank info, so create a custom tiling dword.
3236          * bits 3:0   num_pipes
3237          * bits 7:4   num_banks
3238          * bits 11:8  group_size
3239          * bits 15:12 row_size
3240          */
3241         rdev->config.si.tile_config = 0;
3242         switch (rdev->config.si.num_tile_pipes) {
3243         case 1:
3244                 rdev->config.si.tile_config |= (0 << 0);
3245                 break;
3246         case 2:
3247                 rdev->config.si.tile_config |= (1 << 0);
3248                 break;
3249         case 4:
3250                 rdev->config.si.tile_config |= (2 << 0);
3251                 break;
3252         case 8:
3253         default:
3254                 /* XXX what about 12? */
3255                 rdev->config.si.tile_config |= (3 << 0);
3256                 break;
3257         }
3258         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3259         case 0: /* four banks */
3260                 rdev->config.si.tile_config |= 0 << 4;
3261                 break;
3262         case 1: /* eight banks */
3263                 rdev->config.si.tile_config |= 1 << 4;
3264                 break;
3265         case 2: /* sixteen banks */
3266         default:
3267                 rdev->config.si.tile_config |= 2 << 4;
3268                 break;
3269         }
3270         rdev->config.si.tile_config |=
3271                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3272         rdev->config.si.tile_config |=
3273                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3274
3275         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3276         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3277         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3278         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3279         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3280         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3281         if (rdev->has_uvd) {
3282                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3283                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3284                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3285         }
3286
3287         si_tiling_mode_table_init(rdev);
3288
3289         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3290                     rdev->config.si.max_sh_per_se,
3291                     rdev->config.si.max_backends_per_se);
3292
3293         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3294                      rdev->config.si.max_sh_per_se,
3295                      rdev->config.si.max_cu_per_sh);
3296
3297         rdev->config.si.active_cus = 0;
3298         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3299                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3300                         rdev->config.si.active_cus +=
3301                                 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3302                 }
3303         }
3304
3305         /* set HW defaults for 3D engine */
3306         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3307                                      ROQ_IB2_START(0x2b)));
3308         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3309
3310         sx_debug_1 = RREG32(SX_DEBUG_1);
3311         WREG32(SX_DEBUG_1, sx_debug_1);
3312
3313         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3314
3315         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3316                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3317                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3318                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3319
3320         WREG32(VGT_NUM_INSTANCES, 1);
3321
3322         WREG32(CP_PERFMON_CNTL, 0);
3323
3324         WREG32(SQ_CONFIG, 0);
3325
3326         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3327                                           FORCE_EOV_MAX_REZ_CNT(255)));
3328
3329         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3330                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3331
3332         WREG32(VGT_GS_VERTEX_REUSE, 16);
3333         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3334
3335         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3336         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3337         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3338         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3339         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3340         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3341         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3342         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3343
3344         tmp = RREG32(HDP_MISC_CNTL);
3345         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3346         WREG32(HDP_MISC_CNTL, tmp);
3347
3348         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3349         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3350
3351         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3352
3353         udelay(50);
3354 }
3355
3356 /*
3357  * GPU scratch registers helpers function.
3358  */
3359 static void si_scratch_init(struct radeon_device *rdev)
3360 {
3361         int i;
3362
3363         rdev->scratch.num_reg = 7;
3364         rdev->scratch.reg_base = SCRATCH_REG0;
3365         for (i = 0; i < rdev->scratch.num_reg; i++) {
3366                 rdev->scratch.free[i] = true;
3367                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3368         }
3369 }
3370
3371 void si_fence_ring_emit(struct radeon_device *rdev,
3372                         struct radeon_fence *fence)
3373 {
3374         struct radeon_ring *ring = &rdev->ring[fence->ring];
3375         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3376
3377         /* flush read cache over gart */
3378         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3379         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3380         radeon_ring_write(ring, 0);
3381         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3382         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3383                           PACKET3_TC_ACTION_ENA |
3384                           PACKET3_SH_KCACHE_ACTION_ENA |
3385                           PACKET3_SH_ICACHE_ACTION_ENA);
3386         radeon_ring_write(ring, 0xFFFFFFFF);
3387         radeon_ring_write(ring, 0);
3388         radeon_ring_write(ring, 10); /* poll interval */
3389         /* EVENT_WRITE_EOP - flush caches, send int */
3390         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3391         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3392         radeon_ring_write(ring, lower_32_bits(addr));
3393         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3394         radeon_ring_write(ring, fence->seq);
3395         radeon_ring_write(ring, 0);
3396 }
3397
3398 /*
3399  * IB stuff
3400  */
3401 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3402 {
3403         struct radeon_ring *ring = &rdev->ring[ib->ring];
3404         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3405         u32 header;
3406
3407         if (ib->is_const_ib) {
3408                 /* set switch buffer packet before const IB */
3409                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3410                 radeon_ring_write(ring, 0);
3411
3412                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3413         } else {
3414                 u32 next_rptr;
3415                 if (ring->rptr_save_reg) {
3416                         next_rptr = ring->wptr + 3 + 4 + 8;
3417                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3418                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3419                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3420                         radeon_ring_write(ring, next_rptr);
3421                 } else if (rdev->wb.enabled) {
3422                         next_rptr = ring->wptr + 5 + 4 + 8;
3423                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3424                         radeon_ring_write(ring, (1 << 8));
3425                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3426                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3427                         radeon_ring_write(ring, next_rptr);
3428                 }
3429
3430                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3431         }
3432
3433         radeon_ring_write(ring, header);
3434         radeon_ring_write(ring,
3435 #ifdef __BIG_ENDIAN
3436                           (2 << 0) |
3437 #endif
3438                           (ib->gpu_addr & 0xFFFFFFFC));
3439         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3440         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3441
3442         if (!ib->is_const_ib) {
3443                 /* flush read cache over gart for this vmid */
3444                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3445                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3446                 radeon_ring_write(ring, vm_id);
3447                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3448                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3449                                   PACKET3_TC_ACTION_ENA |
3450                                   PACKET3_SH_KCACHE_ACTION_ENA |
3451                                   PACKET3_SH_ICACHE_ACTION_ENA);
3452                 radeon_ring_write(ring, 0xFFFFFFFF);
3453                 radeon_ring_write(ring, 0);
3454                 radeon_ring_write(ring, 10); /* poll interval */
3455         }
3456 }
3457
3458 /*
3459  * CP.
3460  */
3461 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3462 {
3463         if (enable)
3464                 WREG32(CP_ME_CNTL, 0);
3465         else {
3466                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3467                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3468                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3469                 WREG32(SCRATCH_UMSK, 0);
3470                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3471                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3472                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3473         }
3474         udelay(50);
3475 }
3476
3477 static int si_cp_load_microcode(struct radeon_device *rdev)
3478 {
3479         int i;
3480
3481         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3482                 return -EINVAL;
3483
3484         si_cp_enable(rdev, false);
3485
3486         if (rdev->new_fw) {
3487                 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3488                         (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3489                 const struct gfx_firmware_header_v1_0 *ce_hdr =
3490                         (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3491                 const struct gfx_firmware_header_v1_0 *me_hdr =
3492                         (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3493                 const __le32 *fw_data;
3494                 u32 fw_size;
3495
3496                 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3497                 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3498                 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3499
3500                 /* PFP */
3501                 fw_data = (const __le32 *)
3502                         (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3503                 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3504                 WREG32(CP_PFP_UCODE_ADDR, 0);
3505                 for (i = 0; i < fw_size; i++)
3506                         WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3507                 WREG32(CP_PFP_UCODE_ADDR, 0);
3508
3509                 /* CE */
3510                 fw_data = (const __le32 *)
3511                         (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3512                 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3513                 WREG32(CP_CE_UCODE_ADDR, 0);
3514                 for (i = 0; i < fw_size; i++)
3515                         WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3516                 WREG32(CP_CE_UCODE_ADDR, 0);
3517
3518                 /* ME */
3519                 fw_data = (const __be32 *)
3520                         (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3521                 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3522                 WREG32(CP_ME_RAM_WADDR, 0);
3523                 for (i = 0; i < fw_size; i++)
3524                         WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3525                 WREG32(CP_ME_RAM_WADDR, 0);
3526         } else {
3527                 const __be32 *fw_data;
3528
3529                 /* PFP */
3530                 fw_data = (const __be32 *)rdev->pfp_fw->data;
3531                 WREG32(CP_PFP_UCODE_ADDR, 0);
3532                 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3533                         WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3534                 WREG32(CP_PFP_UCODE_ADDR, 0);
3535
3536                 /* CE */
3537                 fw_data = (const __be32 *)rdev->ce_fw->data;
3538                 WREG32(CP_CE_UCODE_ADDR, 0);
3539                 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3540                         WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3541                 WREG32(CP_CE_UCODE_ADDR, 0);
3542
3543                 /* ME */
3544                 fw_data = (const __be32 *)rdev->me_fw->data;
3545                 WREG32(CP_ME_RAM_WADDR, 0);
3546                 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3547                         WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3548                 WREG32(CP_ME_RAM_WADDR, 0);
3549         }
3550
3551         WREG32(CP_PFP_UCODE_ADDR, 0);
3552         WREG32(CP_CE_UCODE_ADDR, 0);
3553         WREG32(CP_ME_RAM_WADDR, 0);
3554         WREG32(CP_ME_RAM_RADDR, 0);
3555         return 0;
3556 }
3557
3558 static int si_cp_start(struct radeon_device *rdev)
3559 {
3560         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3561         int r, i;
3562
3563         r = radeon_ring_lock(rdev, ring, 7 + 4);
3564         if (r) {
3565                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3566                 return r;
3567         }
3568         /* init the CP */
3569         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3570         radeon_ring_write(ring, 0x1);
3571         radeon_ring_write(ring, 0x0);
3572         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3573         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3574         radeon_ring_write(ring, 0);
3575         radeon_ring_write(ring, 0);
3576
3577         /* init the CE partitions */
3578         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3579         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3580         radeon_ring_write(ring, 0xc000);
3581         radeon_ring_write(ring, 0xe000);
3582         radeon_ring_unlock_commit(rdev, ring, false);
3583
3584         si_cp_enable(rdev, true);
3585
3586         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3587         if (r) {
3588                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3589                 return r;
3590         }
3591
3592         /* setup clear context state */
3593         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3594         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3595
3596         for (i = 0; i < si_default_size; i++)
3597                 radeon_ring_write(ring, si_default_state[i]);
3598
3599         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3600         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3601
3602         /* set clear context state */
3603         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3604         radeon_ring_write(ring, 0);
3605
3606         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3607         radeon_ring_write(ring, 0x00000316);
3608         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3609         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3610
3611         radeon_ring_unlock_commit(rdev, ring, false);
3612
3613         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3614                 ring = &rdev->ring[i];
3615                 r = radeon_ring_lock(rdev, ring, 2);
3616
3617                 /* clear the compute context state */
3618                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3619                 radeon_ring_write(ring, 0);
3620
3621                 radeon_ring_unlock_commit(rdev, ring, false);
3622         }
3623
3624         return 0;
3625 }
3626
3627 static void si_cp_fini(struct radeon_device *rdev)
3628 {
3629         struct radeon_ring *ring;
3630         si_cp_enable(rdev, false);
3631
3632         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3633         radeon_ring_fini(rdev, ring);
3634         radeon_scratch_free(rdev, ring->rptr_save_reg);
3635
3636         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3637         radeon_ring_fini(rdev, ring);
3638         radeon_scratch_free(rdev, ring->rptr_save_reg);
3639
3640         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3641         radeon_ring_fini(rdev, ring);
3642         radeon_scratch_free(rdev, ring->rptr_save_reg);
3643 }
3644
3645 static int si_cp_resume(struct radeon_device *rdev)
3646 {
3647         struct radeon_ring *ring;
3648         u32 tmp;
3649         u32 rb_bufsz;
3650         int r;
3651
3652         si_enable_gui_idle_interrupt(rdev, false);
3653
3654         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3655         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3656
3657         /* Set the write pointer delay */
3658         WREG32(CP_RB_WPTR_DELAY, 0);
3659
3660         WREG32(CP_DEBUG, 0);
3661         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3662
3663         /* ring 0 - compute and gfx */
3664         /* Set ring buffer size */
3665         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3666         rb_bufsz = order_base_2(ring->ring_size / 8);
3667         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3668 #ifdef __BIG_ENDIAN
3669         tmp |= BUF_SWAP_32BIT;
3670 #endif
3671         WREG32(CP_RB0_CNTL, tmp);
3672
3673         /* Initialize the ring buffer's read and write pointers */
3674         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3675         ring->wptr = 0;
3676         WREG32(CP_RB0_WPTR, ring->wptr);
3677
3678         /* set the wb address whether it's enabled or not */
3679         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3680         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3681
3682         if (rdev->wb.enabled)
3683                 WREG32(SCRATCH_UMSK, 0xff);
3684         else {
3685                 tmp |= RB_NO_UPDATE;
3686                 WREG32(SCRATCH_UMSK, 0);
3687         }
3688
3689         mdelay(1);
3690         WREG32(CP_RB0_CNTL, tmp);
3691
3692         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3693
3694         /* ring1  - compute only */
3695         /* Set ring buffer size */
3696         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3697         rb_bufsz = order_base_2(ring->ring_size / 8);
3698         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3699 #ifdef __BIG_ENDIAN
3700         tmp |= BUF_SWAP_32BIT;
3701 #endif
3702         WREG32(CP_RB1_CNTL, tmp);
3703
3704         /* Initialize the ring buffer's read and write pointers */
3705         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3706         ring->wptr = 0;
3707         WREG32(CP_RB1_WPTR, ring->wptr);
3708
3709         /* set the wb address whether it's enabled or not */
3710         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3711         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3712
3713         mdelay(1);
3714         WREG32(CP_RB1_CNTL, tmp);
3715
3716         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3717
3718         /* ring2 - compute only */
3719         /* Set ring buffer size */
3720         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3721         rb_bufsz = order_base_2(ring->ring_size / 8);
3722         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3723 #ifdef __BIG_ENDIAN
3724         tmp |= BUF_SWAP_32BIT;
3725 #endif
3726         WREG32(CP_RB2_CNTL, tmp);
3727
3728         /* Initialize the ring buffer's read and write pointers */
3729         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3730         ring->wptr = 0;
3731         WREG32(CP_RB2_WPTR, ring->wptr);
3732
3733         /* set the wb address whether it's enabled or not */
3734         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3735         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3736
3737         mdelay(1);
3738         WREG32(CP_RB2_CNTL, tmp);
3739
3740         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3741
3742         /* start the rings */
3743         si_cp_start(rdev);
3744         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3745         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3746         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3747         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3748         if (r) {
3749                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3750                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3751                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3752                 return r;
3753         }
3754         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3755         if (r) {
3756                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3757         }
3758         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3759         if (r) {
3760                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3761         }
3762
3763         si_enable_gui_idle_interrupt(rdev, true);
3764
3765         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3766                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3767
3768         return 0;
3769 }
3770
3771 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3772 {
3773         u32 reset_mask = 0;
3774         u32 tmp;
3775
3776         /* GRBM_STATUS */
3777         tmp = RREG32(GRBM_STATUS);
3778         if (tmp & (PA_BUSY | SC_BUSY |
3779                    BCI_BUSY | SX_BUSY |
3780                    TA_BUSY | VGT_BUSY |
3781                    DB_BUSY | CB_BUSY |
3782                    GDS_BUSY | SPI_BUSY |
3783                    IA_BUSY | IA_BUSY_NO_DMA))
3784                 reset_mask |= RADEON_RESET_GFX;
3785
3786         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3787                    CP_BUSY | CP_COHERENCY_BUSY))
3788                 reset_mask |= RADEON_RESET_CP;
3789
3790         if (tmp & GRBM_EE_BUSY)
3791                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3792
3793         /* GRBM_STATUS2 */
3794         tmp = RREG32(GRBM_STATUS2);
3795         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3796                 reset_mask |= RADEON_RESET_RLC;
3797
3798         /* DMA_STATUS_REG 0 */
3799         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3800         if (!(tmp & DMA_IDLE))
3801                 reset_mask |= RADEON_RESET_DMA;
3802
3803         /* DMA_STATUS_REG 1 */
3804         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3805         if (!(tmp & DMA_IDLE))
3806                 reset_mask |= RADEON_RESET_DMA1;
3807
3808         /* SRBM_STATUS2 */
3809         tmp = RREG32(SRBM_STATUS2);
3810         if (tmp & DMA_BUSY)
3811                 reset_mask |= RADEON_RESET_DMA;
3812
3813         if (tmp & DMA1_BUSY)
3814                 reset_mask |= RADEON_RESET_DMA1;
3815
3816         /* SRBM_STATUS */
3817         tmp = RREG32(SRBM_STATUS);
3818
3819         if (tmp & IH_BUSY)
3820                 reset_mask |= RADEON_RESET_IH;
3821
3822         if (tmp & SEM_BUSY)
3823                 reset_mask |= RADEON_RESET_SEM;
3824
3825         if (tmp & GRBM_RQ_PENDING)
3826                 reset_mask |= RADEON_RESET_GRBM;
3827
3828         if (tmp & VMC_BUSY)
3829                 reset_mask |= RADEON_RESET_VMC;
3830
3831         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3832                    MCC_BUSY | MCD_BUSY))
3833                 reset_mask |= RADEON_RESET_MC;
3834
3835         if (evergreen_is_display_hung(rdev))
3836                 reset_mask |= RADEON_RESET_DISPLAY;
3837
3838         /* VM_L2_STATUS */
3839         tmp = RREG32(VM_L2_STATUS);
3840         if (tmp & L2_BUSY)
3841                 reset_mask |= RADEON_RESET_VMC;
3842
3843         /* Skip MC reset as it's mostly likely not hung, just busy */
3844         if (reset_mask & RADEON_RESET_MC) {
3845                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3846                 reset_mask &= ~RADEON_RESET_MC;
3847         }
3848
3849         return reset_mask;
3850 }
3851
3852 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3853 {
3854         struct evergreen_mc_save save;
3855         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3856         u32 tmp;
3857
3858         if (reset_mask == 0)
3859                 return;
3860
3861         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3862
3863         evergreen_print_gpu_status_regs(rdev);
3864         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3865                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3866         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3867                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3868
3869         /* disable PG/CG */
3870         si_fini_pg(rdev);
3871         si_fini_cg(rdev);
3872
3873         /* stop the rlc */
3874         si_rlc_stop(rdev);
3875
3876         /* Disable CP parsing/prefetching */
3877         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3878
3879         if (reset_mask & RADEON_RESET_DMA) {
3880                 /* dma0 */
3881                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3882                 tmp &= ~DMA_RB_ENABLE;
3883                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3884         }
3885         if (reset_mask & RADEON_RESET_DMA1) {
3886                 /* dma1 */
3887                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3888                 tmp &= ~DMA_RB_ENABLE;
3889                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3890         }
3891
3892         udelay(50);
3893
3894         evergreen_mc_stop(rdev, &save);
3895         if (evergreen_mc_wait_for_idle(rdev)) {
3896                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3897         }
3898
3899         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3900                 grbm_soft_reset = SOFT_RESET_CB |
3901                         SOFT_RESET_DB |
3902                         SOFT_RESET_GDS |
3903                         SOFT_RESET_PA |
3904                         SOFT_RESET_SC |
3905                         SOFT_RESET_BCI |
3906                         SOFT_RESET_SPI |
3907                         SOFT_RESET_SX |
3908                         SOFT_RESET_TC |
3909                         SOFT_RESET_TA |
3910                         SOFT_RESET_VGT |
3911                         SOFT_RESET_IA;
3912         }
3913
3914         if (reset_mask & RADEON_RESET_CP) {
3915                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3916
3917                 srbm_soft_reset |= SOFT_RESET_GRBM;
3918         }
3919
3920         if (reset_mask & RADEON_RESET_DMA)
3921                 srbm_soft_reset |= SOFT_RESET_DMA;
3922
3923         if (reset_mask & RADEON_RESET_DMA1)
3924                 srbm_soft_reset |= SOFT_RESET_DMA1;
3925
3926         if (reset_mask & RADEON_RESET_DISPLAY)
3927                 srbm_soft_reset |= SOFT_RESET_DC;
3928
3929         if (reset_mask & RADEON_RESET_RLC)
3930                 grbm_soft_reset |= SOFT_RESET_RLC;
3931
3932         if (reset_mask & RADEON_RESET_SEM)
3933                 srbm_soft_reset |= SOFT_RESET_SEM;
3934
3935         if (reset_mask & RADEON_RESET_IH)
3936                 srbm_soft_reset |= SOFT_RESET_IH;
3937
3938         if (reset_mask & RADEON_RESET_GRBM)
3939                 srbm_soft_reset |= SOFT_RESET_GRBM;
3940
3941         if (reset_mask & RADEON_RESET_VMC)
3942                 srbm_soft_reset |= SOFT_RESET_VMC;
3943
3944         if (reset_mask & RADEON_RESET_MC)
3945                 srbm_soft_reset |= SOFT_RESET_MC;
3946
3947         if (grbm_soft_reset) {
3948                 tmp = RREG32(GRBM_SOFT_RESET);
3949                 tmp |= grbm_soft_reset;
3950                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3951                 WREG32(GRBM_SOFT_RESET, tmp);
3952                 tmp = RREG32(GRBM_SOFT_RESET);
3953
3954                 udelay(50);
3955
3956                 tmp &= ~grbm_soft_reset;
3957                 WREG32(GRBM_SOFT_RESET, tmp);
3958                 tmp = RREG32(GRBM_SOFT_RESET);
3959         }
3960
3961         if (srbm_soft_reset) {
3962                 tmp = RREG32(SRBM_SOFT_RESET);
3963                 tmp |= srbm_soft_reset;
3964                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3965                 WREG32(SRBM_SOFT_RESET, tmp);
3966                 tmp = RREG32(SRBM_SOFT_RESET);
3967
3968                 udelay(50);
3969
3970                 tmp &= ~srbm_soft_reset;
3971                 WREG32(SRBM_SOFT_RESET, tmp);
3972                 tmp = RREG32(SRBM_SOFT_RESET);
3973         }
3974
3975         /* Wait a little for things to settle down */
3976         udelay(50);
3977
3978         evergreen_mc_resume(rdev, &save);
3979         udelay(50);
3980
3981         evergreen_print_gpu_status_regs(rdev);
3982 }
3983
3984 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3985 {
3986         u32 tmp, i;
3987
3988         tmp = RREG32(CG_SPLL_FUNC_CNTL);
3989         tmp |= SPLL_BYPASS_EN;
3990         WREG32(CG_SPLL_FUNC_CNTL, tmp);
3991
3992         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3993         tmp |= SPLL_CTLREQ_CHG;
3994         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3995
3996         for (i = 0; i < rdev->usec_timeout; i++) {
3997                 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3998                         break;
3999                 udelay(1);
4000         }
4001
4002         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
4003         tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4004         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4005
4006         tmp = RREG32(MPLL_CNTL_MODE);
4007         tmp &= ~MPLL_MCLK_SEL;
4008         WREG32(MPLL_CNTL_MODE, tmp);
4009 }
4010
4011 static void si_spll_powerdown(struct radeon_device *rdev)
4012 {
4013         u32 tmp;
4014
4015         tmp = RREG32(SPLL_CNTL_MODE);
4016         tmp |= SPLL_SW_DIR_CONTROL;
4017         WREG32(SPLL_CNTL_MODE, tmp);
4018
4019         tmp = RREG32(CG_SPLL_FUNC_CNTL);
4020         tmp |= SPLL_RESET;
4021         WREG32(CG_SPLL_FUNC_CNTL, tmp);
4022
4023         tmp = RREG32(CG_SPLL_FUNC_CNTL);
4024         tmp |= SPLL_SLEEP;
4025         WREG32(CG_SPLL_FUNC_CNTL, tmp);
4026
4027         tmp = RREG32(SPLL_CNTL_MODE);
4028         tmp &= ~SPLL_SW_DIR_CONTROL;
4029         WREG32(SPLL_CNTL_MODE, tmp);
4030 }
4031
4032 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4033 {
4034         struct evergreen_mc_save save;
4035         u32 tmp, i;
4036
4037         dev_info(rdev->dev, "GPU pci config reset\n");
4038
4039         /* disable dpm? */
4040
4041         /* disable cg/pg */
4042         si_fini_pg(rdev);
4043         si_fini_cg(rdev);
4044
4045         /* Disable CP parsing/prefetching */
4046         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4047         /* dma0 */
4048         tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4049         tmp &= ~DMA_RB_ENABLE;
4050         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4051         /* dma1 */
4052         tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4053         tmp &= ~DMA_RB_ENABLE;
4054         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4055         /* XXX other engines? */
4056
4057         /* halt the rlc, disable cp internal ints */
4058         si_rlc_stop(rdev);
4059
4060         udelay(50);
4061
4062         /* disable mem access */
4063         evergreen_mc_stop(rdev, &save);
4064         if (evergreen_mc_wait_for_idle(rdev)) {
4065                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4066         }
4067
4068         /* set mclk/sclk to bypass */
4069         si_set_clk_bypass_mode(rdev);
4070         /* powerdown spll */
4071         si_spll_powerdown(rdev);
4072         /* disable BM */
4073         pci_clear_master(rdev->pdev);
4074         /* reset */
4075         radeon_pci_config_reset(rdev);
4076         /* wait for asic to come out of reset */
4077         for (i = 0; i < rdev->usec_timeout; i++) {
4078                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4079                         break;
4080                 udelay(1);
4081         }
4082 }
4083
4084 int si_asic_reset(struct radeon_device *rdev)
4085 {
4086         u32 reset_mask;
4087
4088         reset_mask = si_gpu_check_soft_reset(rdev);
4089
4090         if (reset_mask)
4091                 r600_set_bios_scratch_engine_hung(rdev, true);
4092
4093         /* try soft reset */
4094         si_gpu_soft_reset(rdev, reset_mask);
4095
4096         reset_mask = si_gpu_check_soft_reset(rdev);
4097
4098         /* try pci config reset */
4099         if (reset_mask && radeon_hard_reset)
4100                 si_gpu_pci_config_reset(rdev);
4101
4102         reset_mask = si_gpu_check_soft_reset(rdev);
4103
4104         if (!reset_mask)
4105                 r600_set_bios_scratch_engine_hung(rdev, false);
4106
4107         return 0;
4108 }
4109
4110 /**
4111  * si_gfx_is_lockup - Check if the GFX engine is locked up
4112  *
4113  * @rdev: radeon_device pointer
4114  * @ring: radeon_ring structure holding ring information
4115  *
4116  * Check if the GFX engine is locked up.
4117  * Returns true if the engine appears to be locked up, false if not.
4118  */
4119 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4120 {
4121         u32 reset_mask = si_gpu_check_soft_reset(rdev);
4122
4123         if (!(reset_mask & (RADEON_RESET_GFX |
4124                             RADEON_RESET_COMPUTE |
4125                             RADEON_RESET_CP))) {
4126                 radeon_ring_lockup_update(rdev, ring);
4127                 return false;
4128         }
4129         return radeon_ring_test_lockup(rdev, ring);
4130 }
4131
4132 /* MC */
4133 static void si_mc_program(struct radeon_device *rdev)
4134 {
4135         struct evergreen_mc_save save;
4136         u32 tmp;
4137         int i, j;
4138
4139         /* Initialize HDP */
4140         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4141                 WREG32((0x2c14 + j), 0x00000000);
4142                 WREG32((0x2c18 + j), 0x00000000);
4143                 WREG32((0x2c1c + j), 0x00000000);
4144                 WREG32((0x2c20 + j), 0x00000000);
4145                 WREG32((0x2c24 + j), 0x00000000);
4146         }
4147         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4148
4149         evergreen_mc_stop(rdev, &save);
4150         if (radeon_mc_wait_for_idle(rdev)) {
4151                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4152         }
4153         if (!ASIC_IS_NODCE(rdev))
4154                 /* Lockout access through VGA aperture*/
4155                 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4156         /* Update configuration */
4157         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4158                rdev->mc.vram_start >> 12);
4159         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4160                rdev->mc.vram_end >> 12);
4161         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4162                rdev->vram_scratch.gpu_addr >> 12);
4163         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4164         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4165         WREG32(MC_VM_FB_LOCATION, tmp);
4166         /* XXX double check these! */
4167         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4168         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4169         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4170         WREG32(MC_VM_AGP_BASE, 0);
4171         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4172         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4173         if (radeon_mc_wait_for_idle(rdev)) {
4174                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4175         }
4176         evergreen_mc_resume(rdev, &save);
4177         if (!ASIC_IS_NODCE(rdev)) {
4178                 /* we need to own VRAM, so turn off the VGA renderer here
4179                  * to stop it overwriting our objects */
4180                 rv515_vga_render_disable(rdev);
4181         }
4182 }
4183
4184 void si_vram_gtt_location(struct radeon_device *rdev,
4185                           struct radeon_mc *mc)
4186 {
4187         if (mc->mc_vram_size > 0xFFC0000000ULL) {
4188                 /* leave room for at least 1024M GTT */
4189                 dev_warn(rdev->dev, "limiting VRAM\n");
4190                 mc->real_vram_size = 0xFFC0000000ULL;
4191                 mc->mc_vram_size = 0xFFC0000000ULL;
4192         }
4193         radeon_vram_location(rdev, &rdev->mc, 0);
4194         rdev->mc.gtt_base_align = 0;
4195         radeon_gtt_location(rdev, mc);
4196 }
4197
4198 static int si_mc_init(struct radeon_device *rdev)
4199 {
4200         u32 tmp;
4201         int chansize, numchan;
4202
4203         /* Get VRAM informations */
4204         rdev->mc.vram_is_ddr = true;
4205         tmp = RREG32(MC_ARB_RAMCFG);
4206         if (tmp & CHANSIZE_OVERRIDE) {
4207                 chansize = 16;
4208         } else if (tmp & CHANSIZE_MASK) {
4209                 chansize = 64;
4210         } else {
4211                 chansize = 32;
4212         }
4213         tmp = RREG32(MC_SHARED_CHMAP);
4214         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4215         case 0:
4216         default:
4217                 numchan = 1;
4218                 break;
4219         case 1:
4220                 numchan = 2;
4221                 break;
4222         case 2:
4223                 numchan = 4;
4224                 break;
4225         case 3:
4226                 numchan = 8;
4227                 break;
4228         case 4:
4229                 numchan = 3;
4230                 break;
4231         case 5:
4232                 numchan = 6;
4233                 break;
4234         case 6:
4235                 numchan = 10;
4236                 break;
4237         case 7:
4238                 numchan = 12;
4239                 break;
4240         case 8:
4241                 numchan = 16;
4242                 break;
4243         }
4244         rdev->mc.vram_width = numchan * chansize;
4245         /* Could aper size report 0 ? */
4246         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4247         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4248         /* size in MB on si */
4249         tmp = RREG32(CONFIG_MEMSIZE);
4250         /* some boards may have garbage in the upper 16 bits */
4251         if (tmp & 0xffff0000) {
4252                 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4253                 if (tmp & 0xffff)
4254                         tmp &= 0xffff;
4255         }
4256         rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4257         rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4258         rdev->mc.visible_vram_size = rdev->mc.aper_size;
4259         si_vram_gtt_location(rdev, &rdev->mc);
4260         radeon_update_bandwidth_info(rdev);
4261
4262         return 0;
4263 }
4264
4265 /*
4266  * GART
4267  */
4268 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4269 {
4270         /* flush hdp cache */
4271         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4272
4273         /* bits 0-15 are the VM contexts0-15 */
4274         WREG32(VM_INVALIDATE_REQUEST, 1);
4275 }
4276
4277 static int si_pcie_gart_enable(struct radeon_device *rdev)
4278 {
4279         int r, i;
4280
4281         if (rdev->gart.robj == NULL) {
4282                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4283                 return -EINVAL;
4284         }
4285         r = radeon_gart_table_vram_pin(rdev);
4286         if (r)
4287                 return r;
4288         /* Setup TLB control */
4289         WREG32(MC_VM_MX_L1_TLB_CNTL,
4290                (0xA << 7) |
4291                ENABLE_L1_TLB |
4292                ENABLE_L1_FRAGMENT_PROCESSING |
4293                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4294                ENABLE_ADVANCED_DRIVER_MODEL |
4295                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4296         /* Setup L2 cache */
4297         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4298                ENABLE_L2_FRAGMENT_PROCESSING |
4299                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4300                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4301                EFFECTIVE_L2_QUEUE_SIZE(7) |
4302                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4303         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4304         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4305                BANK_SELECT(4) |
4306                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4307         /* setup context0 */
4308         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4309         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4310         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4311         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4312                         (u32)(rdev->dummy_page.addr >> 12));
4313         WREG32(VM_CONTEXT0_CNTL2, 0);
4314         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4315                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4316
4317         WREG32(0x15D4, 0);
4318         WREG32(0x15D8, 0);
4319         WREG32(0x15DC, 0);
4320
4321         /* empty context1-15 */
4322         /* set vm size, must be a multiple of 4 */
4323         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4324         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4325         /* Assign the pt base to something valid for now; the pts used for
4326          * the VMs are determined by the application and setup and assigned
4327          * on the fly in the vm part of radeon_gart.c
4328          */
4329         for (i = 1; i < 16; i++) {
4330                 if (i < 8)
4331                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4332                                rdev->vm_manager.saved_table_addr[i]);
4333                 else
4334                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4335                                rdev->vm_manager.saved_table_addr[i]);
4336         }
4337
4338         /* enable context1-15 */
4339         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4340                (u32)(rdev->dummy_page.addr >> 12));
4341         WREG32(VM_CONTEXT1_CNTL2, 4);
4342         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4343                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4344                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4345                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4346                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4347                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4348                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4349                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4350                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4351                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4352                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4353                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4354                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4355                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4356
4357         si_pcie_gart_tlb_flush(rdev);
4358         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4359                  (unsigned)(rdev->mc.gtt_size >> 20),
4360                  (unsigned long long)rdev->gart.table_addr);
4361         rdev->gart.ready = true;
4362         return 0;
4363 }
4364
4365 static void si_pcie_gart_disable(struct radeon_device *rdev)
4366 {
4367         unsigned i;
4368
4369         for (i = 1; i < 16; ++i) {
4370                 uint32_t reg;
4371                 if (i < 8)
4372                         reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4373                 else
4374                         reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4375                 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4376         }
4377
4378         /* Disable all tables */
4379         WREG32(VM_CONTEXT0_CNTL, 0);
4380         WREG32(VM_CONTEXT1_CNTL, 0);
4381         /* Setup TLB control */
4382         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4383                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4384         /* Setup L2 cache */
4385         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4386                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4387                EFFECTIVE_L2_QUEUE_SIZE(7) |
4388                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4389         WREG32(VM_L2_CNTL2, 0);
4390         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4391                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4392         radeon_gart_table_vram_unpin(rdev);
4393 }
4394
4395 static void si_pcie_gart_fini(struct radeon_device *rdev)
4396 {
4397         si_pcie_gart_disable(rdev);
4398         radeon_gart_table_vram_free(rdev);
4399         radeon_gart_fini(rdev);
4400 }
4401
4402 /* vm parser */
4403 static bool si_vm_reg_valid(u32 reg)
4404 {
4405         /* context regs are fine */
4406         if (reg >= 0x28000)
4407                 return true;
4408
4409         /* check config regs */
4410         switch (reg) {
4411         case GRBM_GFX_INDEX:
4412         case CP_STRMOUT_CNTL:
4413         case VGT_VTX_VECT_EJECT_REG:
4414         case VGT_CACHE_INVALIDATION:
4415         case VGT_ESGS_RING_SIZE:
4416         case VGT_GSVS_RING_SIZE:
4417         case VGT_GS_VERTEX_REUSE:
4418         case VGT_PRIMITIVE_TYPE:
4419         case VGT_INDEX_TYPE:
4420         case VGT_NUM_INDICES:
4421         case VGT_NUM_INSTANCES:
4422         case VGT_TF_RING_SIZE:
4423         case VGT_HS_OFFCHIP_PARAM:
4424         case VGT_TF_MEMORY_BASE:
4425         case PA_CL_ENHANCE:
4426         case PA_SU_LINE_STIPPLE_VALUE:
4427         case PA_SC_LINE_STIPPLE_STATE:
4428         case PA_SC_ENHANCE:
4429         case SQC_CACHES:
4430         case SPI_STATIC_THREAD_MGMT_1:
4431         case SPI_STATIC_THREAD_MGMT_2:
4432         case SPI_STATIC_THREAD_MGMT_3:
4433         case SPI_PS_MAX_WAVE_ID:
4434         case SPI_CONFIG_CNTL:
4435         case SPI_CONFIG_CNTL_1:
4436         case TA_CNTL_AUX:
4437                 return true;
4438         default:
4439                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4440                 return false;
4441         }
4442 }
4443
4444 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4445                                   u32 *ib, struct radeon_cs_packet *pkt)
4446 {
4447         switch (pkt->opcode) {
4448         case PACKET3_NOP:
4449         case PACKET3_SET_BASE:
4450         case PACKET3_SET_CE_DE_COUNTERS:
4451         case PACKET3_LOAD_CONST_RAM:
4452         case PACKET3_WRITE_CONST_RAM:
4453         case PACKET3_WRITE_CONST_RAM_OFFSET:
4454         case PACKET3_DUMP_CONST_RAM:
4455         case PACKET3_INCREMENT_CE_COUNTER:
4456         case PACKET3_WAIT_ON_DE_COUNTER:
4457         case PACKET3_CE_WRITE:
4458                 break;
4459         default:
4460                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4461                 return -EINVAL;
4462         }
4463         return 0;
4464 }
4465
4466 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4467 {
4468         u32 start_reg, reg, i;
4469         u32 command = ib[idx + 4];
4470         u32 info = ib[idx + 1];
4471         u32 idx_value = ib[idx];
4472         if (command & PACKET3_CP_DMA_CMD_SAS) {
4473                 /* src address space is register */
4474                 if (((info & 0x60000000) >> 29) == 0) {
4475                         start_reg = idx_value << 2;
4476                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
4477                                 reg = start_reg;
4478                                 if (!si_vm_reg_valid(reg)) {
4479                                         DRM_ERROR("CP DMA Bad SRC register\n");
4480                                         return -EINVAL;
4481                                 }
4482                         } else {
4483                                 for (i = 0; i < (command & 0x1fffff); i++) {
4484                                         reg = start_reg + (4 * i);
4485                                         if (!si_vm_reg_valid(reg)) {
4486                                                 DRM_ERROR("CP DMA Bad SRC register\n");
4487                                                 return -EINVAL;
4488                                         }
4489                                 }
4490                         }
4491                 }
4492         }
4493         if (command & PACKET3_CP_DMA_CMD_DAS) {
4494                 /* dst address space is register */
4495                 if (((info & 0x00300000) >> 20) == 0) {
4496                         start_reg = ib[idx + 2];
4497                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
4498                                 reg = start_reg;
4499                                 if (!si_vm_reg_valid(reg)) {
4500                                         DRM_ERROR("CP DMA Bad DST register\n");
4501                                         return -EINVAL;
4502                                 }
4503                         } else {
4504                                 for (i = 0; i < (command & 0x1fffff); i++) {
4505                                         reg = start_reg + (4 * i);
4506                                 if (!si_vm_reg_valid(reg)) {
4507                                                 DRM_ERROR("CP DMA Bad DST register\n");
4508                                                 return -EINVAL;
4509                                         }
4510                                 }
4511                         }
4512                 }
4513         }
4514         return 0;
4515 }
4516
4517 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4518                                    u32 *ib, struct radeon_cs_packet *pkt)
4519 {
4520         int r;
4521         u32 idx = pkt->idx + 1;
4522         u32 idx_value = ib[idx];
4523         u32 start_reg, end_reg, reg, i;
4524
4525         switch (pkt->opcode) {
4526         case PACKET3_NOP:
4527         case PACKET3_SET_BASE:
4528         case PACKET3_CLEAR_STATE:
4529         case PACKET3_INDEX_BUFFER_SIZE:
4530         case PACKET3_DISPATCH_DIRECT:
4531         case PACKET3_DISPATCH_INDIRECT:
4532         case PACKET3_ALLOC_GDS:
4533         case PACKET3_WRITE_GDS_RAM:
4534         case PACKET3_ATOMIC_GDS:
4535         case PACKET3_ATOMIC:
4536         case PACKET3_OCCLUSION_QUERY:
4537         case PACKET3_SET_PREDICATION:
4538         case PACKET3_COND_EXEC:
4539         case PACKET3_PRED_EXEC:
4540         case PACKET3_DRAW_INDIRECT:
4541         case PACKET3_DRAW_INDEX_INDIRECT:
4542         case PACKET3_INDEX_BASE:
4543         case PACKET3_DRAW_INDEX_2:
4544         case PACKET3_CONTEXT_CONTROL:
4545         case PACKET3_INDEX_TYPE:
4546         case PACKET3_DRAW_INDIRECT_MULTI:
4547         case PACKET3_DRAW_INDEX_AUTO:
4548         case PACKET3_DRAW_INDEX_IMMD:
4549         case PACKET3_NUM_INSTANCES:
4550         case PACKET3_DRAW_INDEX_MULTI_AUTO:
4551         case PACKET3_STRMOUT_BUFFER_UPDATE:
4552         case PACKET3_DRAW_INDEX_OFFSET_2:
4553         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4554         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4555         case PACKET3_MPEG_INDEX:
4556         case PACKET3_WAIT_REG_MEM:
4557         case PACKET3_MEM_WRITE:
4558         case PACKET3_PFP_SYNC_ME:
4559         case PACKET3_SURFACE_SYNC:
4560         case PACKET3_EVENT_WRITE:
4561         case PACKET3_EVENT_WRITE_EOP:
4562         case PACKET3_EVENT_WRITE_EOS:
4563         case PACKET3_SET_CONTEXT_REG:
4564         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4565         case PACKET3_SET_SH_REG:
4566         case PACKET3_SET_SH_REG_OFFSET:
4567         case PACKET3_INCREMENT_DE_COUNTER:
4568         case PACKET3_WAIT_ON_CE_COUNTER:
4569         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4570         case PACKET3_ME_WRITE:
4571                 break;
4572         case PACKET3_COPY_DATA:
4573                 if ((idx_value & 0xf00) == 0) {
4574                         reg = ib[idx + 3] * 4;
4575                         if (!si_vm_reg_valid(reg))
4576                                 return -EINVAL;
4577                 }
4578                 break;
4579         case PACKET3_WRITE_DATA:
4580                 if ((idx_value & 0xf00) == 0) {
4581                         start_reg = ib[idx + 1] * 4;
4582                         if (idx_value & 0x10000) {
4583                                 if (!si_vm_reg_valid(start_reg))
4584                                         return -EINVAL;
4585                         } else {
4586                                 for (i = 0; i < (pkt->count - 2); i++) {
4587                                         reg = start_reg + (4 * i);
4588                                         if (!si_vm_reg_valid(reg))
4589                                                 return -EINVAL;
4590                                 }
4591                         }
4592                 }
4593                 break;
4594         case PACKET3_COND_WRITE:
4595                 if (idx_value & 0x100) {
4596                         reg = ib[idx + 5] * 4;
4597                         if (!si_vm_reg_valid(reg))
4598                                 return -EINVAL;
4599                 }
4600                 break;
4601         case PACKET3_COPY_DW:
4602                 if (idx_value & 0x2) {
4603                         reg = ib[idx + 3] * 4;
4604                         if (!si_vm_reg_valid(reg))
4605                                 return -EINVAL;
4606                 }
4607                 break;
4608         case PACKET3_SET_CONFIG_REG:
4609                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4610                 end_reg = 4 * pkt->count + start_reg - 4;
4611                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4612                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4613                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4614                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4615                         return -EINVAL;
4616                 }
4617                 for (i = 0; i < pkt->count; i++) {
4618                         reg = start_reg + (4 * i);
4619                         if (!si_vm_reg_valid(reg))
4620                                 return -EINVAL;
4621                 }
4622                 break;
4623         case PACKET3_CP_DMA:
4624                 r = si_vm_packet3_cp_dma_check(ib, idx);
4625                 if (r)
4626                         return r;
4627                 break;
4628         default:
4629                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4630                 return -EINVAL;
4631         }
4632         return 0;
4633 }
4634
4635 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4636                                        u32 *ib, struct radeon_cs_packet *pkt)
4637 {
4638         int r;
4639         u32 idx = pkt->idx + 1;
4640         u32 idx_value = ib[idx];
4641         u32 start_reg, reg, i;
4642
4643         switch (pkt->opcode) {
4644         case PACKET3_NOP:
4645         case PACKET3_SET_BASE:
4646         case PACKET3_CLEAR_STATE:
4647         case PACKET3_DISPATCH_DIRECT:
4648         case PACKET3_DISPATCH_INDIRECT:
4649         case PACKET3_ALLOC_GDS:
4650         case PACKET3_WRITE_GDS_RAM:
4651         case PACKET3_ATOMIC_GDS:
4652         case PACKET3_ATOMIC:
4653         case PACKET3_OCCLUSION_QUERY:
4654         case PACKET3_SET_PREDICATION:
4655         case PACKET3_COND_EXEC:
4656         case PACKET3_PRED_EXEC:
4657         case PACKET3_CONTEXT_CONTROL:
4658         case PACKET3_STRMOUT_BUFFER_UPDATE:
4659         case PACKET3_WAIT_REG_MEM:
4660         case PACKET3_MEM_WRITE:
4661         case PACKET3_PFP_SYNC_ME:
4662         case PACKET3_SURFACE_SYNC:
4663         case PACKET3_EVENT_WRITE:
4664         case PACKET3_EVENT_WRITE_EOP:
4665         case PACKET3_EVENT_WRITE_EOS:
4666         case PACKET3_SET_CONTEXT_REG:
4667         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4668         case PACKET3_SET_SH_REG:
4669         case PACKET3_SET_SH_REG_OFFSET:
4670         case PACKET3_INCREMENT_DE_COUNTER:
4671         case PACKET3_WAIT_ON_CE_COUNTER:
4672         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4673         case PACKET3_ME_WRITE:
4674                 break;
4675         case PACKET3_COPY_DATA:
4676                 if ((idx_value & 0xf00) == 0) {
4677                         reg = ib[idx + 3] * 4;
4678                         if (!si_vm_reg_valid(reg))
4679                                 return -EINVAL;
4680                 }
4681                 break;
4682         case PACKET3_WRITE_DATA:
4683                 if ((idx_value & 0xf00) == 0) {
4684                         start_reg = ib[idx + 1] * 4;
4685                         if (idx_value & 0x10000) {
4686                                 if (!si_vm_reg_valid(start_reg))
4687                                         return -EINVAL;
4688                         } else {
4689                                 for (i = 0; i < (pkt->count - 2); i++) {
4690                                         reg = start_reg + (4 * i);
4691                                         if (!si_vm_reg_valid(reg))
4692                                                 return -EINVAL;
4693                                 }
4694                         }
4695                 }
4696                 break;
4697         case PACKET3_COND_WRITE:
4698                 if (idx_value & 0x100) {
4699                         reg = ib[idx + 5] * 4;
4700                         if (!si_vm_reg_valid(reg))
4701                                 return -EINVAL;
4702                 }
4703                 break;
4704         case PACKET3_COPY_DW:
4705                 if (idx_value & 0x2) {
4706                         reg = ib[idx + 3] * 4;
4707                         if (!si_vm_reg_valid(reg))
4708                                 return -EINVAL;
4709                 }
4710                 break;
4711         case PACKET3_CP_DMA:
4712                 r = si_vm_packet3_cp_dma_check(ib, idx);
4713                 if (r)
4714                         return r;
4715                 break;
4716         default:
4717                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4718                 return -EINVAL;
4719         }
4720         return 0;
4721 }
4722
4723 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4724 {
4725         int ret = 0;
4726         u32 idx = 0, i;
4727         struct radeon_cs_packet pkt;
4728
4729         do {
4730                 pkt.idx = idx;
4731                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4732                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4733                 pkt.one_reg_wr = 0;
4734                 switch (pkt.type) {
4735                 case RADEON_PACKET_TYPE0:
4736                         dev_err(rdev->dev, "Packet0 not allowed!\n");
4737                         ret = -EINVAL;
4738                         break;
4739                 case RADEON_PACKET_TYPE2:
4740                         idx += 1;
4741                         break;
4742                 case RADEON_PACKET_TYPE3:
4743                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4744                         if (ib->is_const_ib)
4745                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4746                         else {
4747                                 switch (ib->ring) {
4748                                 case RADEON_RING_TYPE_GFX_INDEX:
4749                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4750                                         break;
4751                                 case CAYMAN_RING_TYPE_CP1_INDEX:
4752                                 case CAYMAN_RING_TYPE_CP2_INDEX:
4753                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4754                                         break;
4755                                 default:
4756                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4757                                         ret = -EINVAL;
4758                                         break;
4759                                 }
4760                         }
4761                         idx += pkt.count + 2;
4762                         break;
4763                 default:
4764                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4765                         ret = -EINVAL;
4766                         break;
4767                 }
4768                 if (ret) {
4769                         for (i = 0; i < ib->length_dw; i++) {
4770                                 if (i == idx)
4771                                         printk("\t0x%08x <---\n", ib->ptr[i]);
4772                                 else
4773                                         printk("\t0x%08x\n", ib->ptr[i]);
4774                         }
4775                         break;
4776                 }
4777         } while (idx < ib->length_dw);
4778
4779         return ret;
4780 }
4781
4782 /*
4783  * vm
4784  */
4785 int si_vm_init(struct radeon_device *rdev)
4786 {
4787         /* number of VMs */
4788         rdev->vm_manager.nvm = 16;
4789         /* base offset of vram pages */
4790         rdev->vm_manager.vram_base_offset = 0;
4791
4792         return 0;
4793 }
4794
4795 void si_vm_fini(struct radeon_device *rdev)
4796 {
4797 }
4798
4799 /**
4800  * si_vm_decode_fault - print human readable fault info
4801  *
4802  * @rdev: radeon_device pointer
4803  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4804  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4805  *
4806  * Print human readable fault information (SI).
4807  */
4808 static void si_vm_decode_fault(struct radeon_device *rdev,
4809                                u32 status, u32 addr)
4810 {
4811         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4812         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4813         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4814         char *block;
4815
4816         if (rdev->family == CHIP_TAHITI) {
4817                 switch (mc_id) {
4818                 case 160:
4819                 case 144:
4820                 case 96:
4821                 case 80:
4822                 case 224:
4823                 case 208:
4824                 case 32:
4825                 case 16:
4826                         block = "CB";
4827                         break;
4828                 case 161:
4829                 case 145:
4830                 case 97:
4831                 case 81:
4832                 case 225:
4833                 case 209:
4834                 case 33:
4835                 case 17:
4836                         block = "CB_FMASK";
4837                         break;
4838                 case 162:
4839                 case 146:
4840                 case 98:
4841                 case 82:
4842                 case 226:
4843                 case 210:
4844                 case 34:
4845                 case 18:
4846                         block = "CB_CMASK";
4847                         break;
4848                 case 163:
4849                 case 147:
4850                 case 99:
4851                 case 83:
4852                 case 227:
4853                 case 211:
4854                 case 35:
4855                 case 19:
4856                         block = "CB_IMMED";
4857                         break;
4858                 case 164:
4859                 case 148:
4860                 case 100:
4861                 case 84:
4862                 case 228:
4863                 case 212:
4864                 case 36:
4865                 case 20:
4866                         block = "DB";
4867                         break;
4868                 case 165:
4869                 case 149:
4870                 case 101:
4871                 case 85:
4872                 case 229:
4873                 case 213:
4874                 case 37:
4875                 case 21:
4876                         block = "DB_HTILE";
4877                         break;
4878                 case 167:
4879                 case 151:
4880                 case 103:
4881                 case 87:
4882                 case 231:
4883                 case 215:
4884                 case 39:
4885                 case 23:
4886                         block = "DB_STEN";
4887                         break;
4888                 case 72:
4889                 case 68:
4890                 case 64:
4891                 case 8:
4892                 case 4:
4893                 case 0:
4894                 case 136:
4895                 case 132:
4896                 case 128:
4897                 case 200:
4898                 case 196:
4899                 case 192:
4900                         block = "TC";
4901                         break;
4902                 case 112:
4903                 case 48:
4904                         block = "CP";
4905                         break;
4906                 case 49:
4907                 case 177:
4908                 case 50:
4909                 case 178:
4910                         block = "SH";
4911                         break;
4912                 case 53:
4913                 case 190:
4914                         block = "VGT";
4915                         break;
4916                 case 117:
4917                         block = "IH";
4918                         break;
4919                 case 51:
4920                 case 115:
4921                         block = "RLC";
4922                         break;
4923                 case 119:
4924                 case 183:
4925                         block = "DMA0";
4926                         break;
4927                 case 61:
4928                         block = "DMA1";
4929                         break;
4930                 case 248:
4931                 case 120:
4932                         block = "HDP";
4933                         break;
4934                 default:
4935                         block = "unknown";
4936                         break;
4937                 }
4938         } else {
4939                 switch (mc_id) {
4940                 case 32:
4941                 case 16:
4942                 case 96:
4943                 case 80:
4944                 case 160:
4945                 case 144:
4946                 case 224:
4947                 case 208:
4948                         block = "CB";
4949                         break;
4950                 case 33:
4951                 case 17:
4952                 case 97:
4953                 case 81:
4954                 case 161:
4955                 case 145:
4956                 case 225:
4957                 case 209:
4958                         block = "CB_FMASK";
4959                         break;
4960                 case 34:
4961                 case 18:
4962                 case 98:
4963                 case 82:
4964                 case 162:
4965                 case 146:
4966                 case 226:
4967                 case 210:
4968                         block = "CB_CMASK";
4969                         break;
4970                 case 35:
4971                 case 19:
4972                 case 99:
4973                 case 83:
4974                 case 163:
4975                 case 147:
4976                 case 227:
4977                 case 211:
4978                         block = "CB_IMMED";
4979                         break;
4980                 case 36:
4981                 case 20:
4982                 case 100:
4983                 case 84:
4984                 case 164:
4985                 case 148:
4986                 case 228:
4987                 case 212:
4988                         block = "DB";
4989                         break;
4990                 case 37:
4991                 case 21:
4992                 case 101:
4993                 case 85:
4994                 case 165:
4995                 case 149:
4996                 case 229:
4997                 case 213:
4998                         block = "DB_HTILE";
4999                         break;
5000                 case 39:
5001                 case 23:
5002                 case 103:
5003                 case 87:
5004                 case 167:
5005                 case 151:
5006                 case 231:
5007                 case 215:
5008                         block = "DB_STEN";
5009                         break;
5010                 case 72:
5011                 case 68:
5012                 case 8:
5013                 case 4:
5014                 case 136:
5015                 case 132:
5016                 case 200:
5017                 case 196:
5018                         block = "TC";
5019                         break;
5020                 case 112:
5021                 case 48:
5022                         block = "CP";
5023                         break;
5024                 case 49:
5025                 case 177:
5026                 case 50:
5027                 case 178:
5028                         block = "SH";
5029                         break;
5030                 case 53:
5031                         block = "VGT";
5032                         break;
5033                 case 117:
5034                         block = "IH";
5035                         break;
5036                 case 51:
5037                 case 115:
5038                         block = "RLC";
5039                         break;
5040                 case 119:
5041                 case 183:
5042                         block = "DMA0";
5043                         break;
5044                 case 61:
5045                         block = "DMA1";
5046                         break;
5047                 case 248:
5048                 case 120:
5049                         block = "HDP";
5050                         break;
5051                 default:
5052                         block = "unknown";
5053                         break;
5054                 }
5055         }
5056
5057         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5058                protections, vmid, addr,
5059                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5060                block, mc_id);
5061 }
5062
5063 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5064                  unsigned vm_id, uint64_t pd_addr)
5065 {
5066         /* write new base address */
5067         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5068         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5069                                  WRITE_DATA_DST_SEL(0)));
5070
5071         if (vm_id < 8) {
5072                 radeon_ring_write(ring,
5073                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5074         } else {
5075                 radeon_ring_write(ring,
5076                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5077         }
5078         radeon_ring_write(ring, 0);
5079         radeon_ring_write(ring, pd_addr >> 12);
5080
5081         /* flush hdp cache */
5082         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5083         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5084                                  WRITE_DATA_DST_SEL(0)));
5085         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5086         radeon_ring_write(ring, 0);
5087         radeon_ring_write(ring, 0x1);
5088
5089         /* bits 0-15 are the VM contexts0-15 */
5090         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5091         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5092                                  WRITE_DATA_DST_SEL(0)));
5093         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5094         radeon_ring_write(ring, 0);
5095         radeon_ring_write(ring, 1 << vm_id);
5096
5097         /* wait for the invalidate to complete */
5098         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5099         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5100                                  WAIT_REG_MEM_ENGINE(0))); /* me */
5101         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5102         radeon_ring_write(ring, 0);
5103         radeon_ring_write(ring, 0); /* ref */
5104         radeon_ring_write(ring, 0); /* mask */
5105         radeon_ring_write(ring, 0x20); /* poll interval */
5106
5107         /* sync PFP to ME, otherwise we might get invalid PFP reads */
5108         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5109         radeon_ring_write(ring, 0x0);
5110 }
5111
5112 /*
5113  *  Power and clock gating
5114  */
5115 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5116 {
5117         int i;
5118
5119         for (i = 0; i < rdev->usec_timeout; i++) {
5120                 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5121                         break;
5122                 udelay(1);
5123         }
5124
5125         for (i = 0; i < rdev->usec_timeout; i++) {
5126                 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5127                         break;
5128                 udelay(1);
5129         }
5130 }
5131
5132 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5133                                          bool enable)
5134 {
5135         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5136         u32 mask;
5137         int i;
5138
5139         if (enable)
5140                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5141         else
5142                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5143         WREG32(CP_INT_CNTL_RING0, tmp);
5144
5145         if (!enable) {
5146                 /* read a gfx register */
5147                 tmp = RREG32(DB_DEPTH_INFO);
5148
5149                 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5150                 for (i = 0; i < rdev->usec_timeout; i++) {
5151                         if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5152                                 break;
5153                         udelay(1);
5154                 }
5155         }
5156 }
5157
5158 static void si_set_uvd_dcm(struct radeon_device *rdev,
5159                            bool sw_mode)
5160 {
5161         u32 tmp, tmp2;
5162
5163         tmp = RREG32(UVD_CGC_CTRL);
5164         tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5165         tmp |= DCM | CG_DT(1) | CLK_OD(4);
5166
5167         if (sw_mode) {
5168                 tmp &= ~0x7ffff800;
5169                 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5170         } else {
5171                 tmp |= 0x7ffff800;
5172                 tmp2 = 0;
5173         }
5174
5175         WREG32(UVD_CGC_CTRL, tmp);
5176         WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5177 }
5178
5179 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5180 {
5181         bool hw_mode = true;
5182
5183         if (hw_mode) {
5184                 si_set_uvd_dcm(rdev, false);
5185         } else {
5186                 u32 tmp = RREG32(UVD_CGC_CTRL);
5187                 tmp &= ~DCM;
5188                 WREG32(UVD_CGC_CTRL, tmp);
5189         }
5190 }
5191
5192 static u32 si_halt_rlc(struct radeon_device *rdev)
5193 {
5194         u32 data, orig;
5195
5196         orig = data = RREG32(RLC_CNTL);
5197
5198         if (data & RLC_ENABLE) {
5199                 data &= ~RLC_ENABLE;
5200                 WREG32(RLC_CNTL, data);
5201
5202                 si_wait_for_rlc_serdes(rdev);
5203         }
5204
5205         return orig;
5206 }
5207
5208 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5209 {
5210         u32 tmp;
5211
5212         tmp = RREG32(RLC_CNTL);
5213         if (tmp != rlc)
5214                 WREG32(RLC_CNTL, rlc);
5215 }
5216
5217 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5218 {
5219         u32 data, orig;
5220
5221         orig = data = RREG32(DMA_PG);
5222         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5223                 data |= PG_CNTL_ENABLE;
5224         else
5225                 data &= ~PG_CNTL_ENABLE;
5226         if (orig != data)
5227                 WREG32(DMA_PG, data);
5228 }
5229
5230 static void si_init_dma_pg(struct radeon_device *rdev)
5231 {
5232         u32 tmp;
5233
5234         WREG32(DMA_PGFSM_WRITE,  0x00002000);
5235         WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5236
5237         for (tmp = 0; tmp < 5; tmp++)
5238                 WREG32(DMA_PGFSM_WRITE, 0);
5239 }
5240
5241 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5242                                bool enable)
5243 {
5244         u32 tmp;
5245
5246         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5247                 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5248                 WREG32(RLC_TTOP_D, tmp);
5249
5250                 tmp = RREG32(RLC_PG_CNTL);
5251                 tmp |= GFX_PG_ENABLE;
5252                 WREG32(RLC_PG_CNTL, tmp);
5253
5254                 tmp = RREG32(RLC_AUTO_PG_CTRL);
5255                 tmp |= AUTO_PG_EN;
5256                 WREG32(RLC_AUTO_PG_CTRL, tmp);
5257         } else {
5258                 tmp = RREG32(RLC_AUTO_PG_CTRL);
5259                 tmp &= ~AUTO_PG_EN;
5260                 WREG32(RLC_AUTO_PG_CTRL, tmp);
5261
5262                 tmp = RREG32(DB_RENDER_CONTROL);
5263         }
5264 }
5265
5266 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5267 {
5268         u32 tmp;
5269
5270         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5271
5272         tmp = RREG32(RLC_PG_CNTL);
5273         tmp |= GFX_PG_SRC;
5274         WREG32(RLC_PG_CNTL, tmp);
5275
5276         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5277
5278         tmp = RREG32(RLC_AUTO_PG_CTRL);
5279
5280         tmp &= ~GRBM_REG_SGIT_MASK;
5281         tmp |= GRBM_REG_SGIT(0x700);
5282         tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5283         WREG32(RLC_AUTO_PG_CTRL, tmp);
5284 }
5285
5286 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5287 {
5288         u32 mask = 0, tmp, tmp1;
5289         int i;
5290
5291         si_select_se_sh(rdev, se, sh);
5292         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5293         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5294         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5295
5296         tmp &= 0xffff0000;
5297
5298         tmp |= tmp1;
5299         tmp >>= 16;
5300
5301         for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5302                 mask <<= 1;
5303                 mask |= 1;
5304         }
5305
5306         return (~tmp) & mask;
5307 }
5308
5309 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5310 {
5311         u32 i, j, k, active_cu_number = 0;
5312         u32 mask, counter, cu_bitmap;
5313         u32 tmp = 0;
5314
5315         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5316                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5317                         mask = 1;
5318                         cu_bitmap = 0;
5319                         counter  = 0;
5320                         for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5321                                 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5322                                         if (counter < 2)
5323                                                 cu_bitmap |= mask;
5324                                         counter++;
5325                                 }
5326                                 mask <<= 1;
5327                         }
5328
5329                         active_cu_number += counter;
5330                         tmp |= (cu_bitmap << (i * 16 + j * 8));
5331                 }
5332         }
5333
5334         WREG32(RLC_PG_AO_CU_MASK, tmp);
5335
5336         tmp = RREG32(RLC_MAX_PG_CU);
5337         tmp &= ~MAX_PU_CU_MASK;
5338         tmp |= MAX_PU_CU(active_cu_number);
5339         WREG32(RLC_MAX_PG_CU, tmp);
5340 }
5341
5342 static void si_enable_cgcg(struct radeon_device *rdev,
5343                            bool enable)
5344 {
5345         u32 data, orig, tmp;
5346
5347         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5348
5349         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5350                 si_enable_gui_idle_interrupt(rdev, true);
5351
5352                 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5353
5354                 tmp = si_halt_rlc(rdev);
5355
5356                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5357                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5358                 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5359
5360                 si_wait_for_rlc_serdes(rdev);
5361
5362                 si_update_rlc(rdev, tmp);
5363
5364                 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5365
5366                 data |= CGCG_EN | CGLS_EN;
5367         } else {
5368                 si_enable_gui_idle_interrupt(rdev, false);
5369
5370                 RREG32(CB_CGTT_SCLK_CTRL);
5371                 RREG32(CB_CGTT_SCLK_CTRL);
5372                 RREG32(CB_CGTT_SCLK_CTRL);
5373                 RREG32(CB_CGTT_SCLK_CTRL);
5374
5375                 data &= ~(CGCG_EN | CGLS_EN);
5376         }
5377
5378         if (orig != data)
5379                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5380 }
5381
5382 static void si_enable_mgcg(struct radeon_device *rdev,
5383                            bool enable)
5384 {
5385         u32 data, orig, tmp = 0;
5386
5387         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5388                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5389                 data = 0x96940200;
5390                 if (orig != data)
5391                         WREG32(CGTS_SM_CTRL_REG, data);
5392
5393                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5394                         orig = data = RREG32(CP_MEM_SLP_CNTL);
5395                         data |= CP_MEM_LS_EN;
5396                         if (orig != data)
5397                                 WREG32(CP_MEM_SLP_CNTL, data);
5398                 }
5399
5400                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5401                 data &= 0xffffffc0;
5402                 if (orig != data)
5403                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5404
5405                 tmp = si_halt_rlc(rdev);
5406
5407                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5408                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5409                 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5410
5411                 si_update_rlc(rdev, tmp);
5412         } else {
5413                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5414                 data |= 0x00000003;
5415                 if (orig != data)
5416                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5417
5418                 data = RREG32(CP_MEM_SLP_CNTL);
5419                 if (data & CP_MEM_LS_EN) {
5420                         data &= ~CP_MEM_LS_EN;
5421                         WREG32(CP_MEM_SLP_CNTL, data);
5422                 }
5423                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5424                 data |= LS_OVERRIDE | OVERRIDE;
5425                 if (orig != data)
5426                         WREG32(CGTS_SM_CTRL_REG, data);
5427
5428                 tmp = si_halt_rlc(rdev);
5429
5430                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5431                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5432                 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5433
5434                 si_update_rlc(rdev, tmp);
5435         }
5436 }
5437
5438 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5439                                bool enable)
5440 {
5441         u32 orig, data, tmp;
5442
5443         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5444                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5445                 tmp |= 0x3fff;
5446                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5447
5448                 orig = data = RREG32(UVD_CGC_CTRL);
5449                 data |= DCM;
5450                 if (orig != data)
5451                         WREG32(UVD_CGC_CTRL, data);
5452
5453                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5454                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5455         } else {
5456                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5457                 tmp &= ~0x3fff;
5458                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5459
5460                 orig = data = RREG32(UVD_CGC_CTRL);
5461                 data &= ~DCM;
5462                 if (orig != data)
5463                         WREG32(UVD_CGC_CTRL, data);
5464
5465                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5466                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5467         }
5468 }
5469
5470 static const u32 mc_cg_registers[] =
5471 {
5472         MC_HUB_MISC_HUB_CG,
5473         MC_HUB_MISC_SIP_CG,
5474         MC_HUB_MISC_VM_CG,
5475         MC_XPB_CLK_GAT,
5476         ATC_MISC_CG,
5477         MC_CITF_MISC_WR_CG,
5478         MC_CITF_MISC_RD_CG,
5479         MC_CITF_MISC_VM_CG,
5480         VM_L2_CG,
5481 };
5482
5483 static void si_enable_mc_ls(struct radeon_device *rdev,
5484                             bool enable)
5485 {
5486         int i;
5487         u32 orig, data;
5488
5489         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5490                 orig = data = RREG32(mc_cg_registers[i]);
5491                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5492                         data |= MC_LS_ENABLE;
5493                 else
5494                         data &= ~MC_LS_ENABLE;
5495                 if (data != orig)
5496                         WREG32(mc_cg_registers[i], data);
5497         }
5498 }
5499
5500 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5501                                bool enable)
5502 {
5503         int i;
5504         u32 orig, data;
5505
5506         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5507                 orig = data = RREG32(mc_cg_registers[i]);
5508                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5509                         data |= MC_CG_ENABLE;
5510                 else
5511                         data &= ~MC_CG_ENABLE;
5512                 if (data != orig)
5513                         WREG32(mc_cg_registers[i], data);
5514         }
5515 }
5516
5517 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5518                                bool enable)
5519 {
5520         u32 orig, data, offset;
5521         int i;
5522
5523         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5524                 for (i = 0; i < 2; i++) {
5525                         if (i == 0)
5526                                 offset = DMA0_REGISTER_OFFSET;
5527                         else
5528                                 offset = DMA1_REGISTER_OFFSET;
5529                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5530                         data &= ~MEM_POWER_OVERRIDE;
5531                         if (data != orig)
5532                                 WREG32(DMA_POWER_CNTL + offset, data);
5533                         WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5534                 }
5535         } else {
5536                 for (i = 0; i < 2; i++) {
5537                         if (i == 0)
5538                                 offset = DMA0_REGISTER_OFFSET;
5539                         else
5540                                 offset = DMA1_REGISTER_OFFSET;
5541                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5542                         data |= MEM_POWER_OVERRIDE;
5543                         if (data != orig)
5544                                 WREG32(DMA_POWER_CNTL + offset, data);
5545
5546                         orig = data = RREG32(DMA_CLK_CTRL + offset);
5547                         data = 0xff000000;
5548                         if (data != orig)
5549                                 WREG32(DMA_CLK_CTRL + offset, data);
5550                 }
5551         }
5552 }
5553
5554 static void si_enable_bif_mgls(struct radeon_device *rdev,
5555                                bool enable)
5556 {
5557         u32 orig, data;
5558
5559         orig = data = RREG32_PCIE(PCIE_CNTL2);
5560
5561         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5562                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5563                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5564         else
5565                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5566                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5567
5568         if (orig != data)
5569                 WREG32_PCIE(PCIE_CNTL2, data);
5570 }
5571
5572 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5573                                bool enable)
5574 {
5575         u32 orig, data;
5576
5577         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5578
5579         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5580                 data &= ~CLOCK_GATING_DIS;
5581         else
5582                 data |= CLOCK_GATING_DIS;
5583
5584         if (orig != data)
5585                 WREG32(HDP_HOST_PATH_CNTL, data);
5586 }
5587
5588 static void si_enable_hdp_ls(struct radeon_device *rdev,
5589                              bool enable)
5590 {
5591         u32 orig, data;
5592
5593         orig = data = RREG32(HDP_MEM_POWER_LS);
5594
5595         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5596                 data |= HDP_LS_ENABLE;
5597         else
5598                 data &= ~HDP_LS_ENABLE;
5599
5600         if (orig != data)
5601                 WREG32(HDP_MEM_POWER_LS, data);
5602 }
5603
5604 static void si_update_cg(struct radeon_device *rdev,
5605                          u32 block, bool enable)
5606 {
5607         if (block & RADEON_CG_BLOCK_GFX) {
5608                 si_enable_gui_idle_interrupt(rdev, false);
5609                 /* order matters! */
5610                 if (enable) {
5611                         si_enable_mgcg(rdev, true);
5612                         si_enable_cgcg(rdev, true);
5613                 } else {
5614                         si_enable_cgcg(rdev, false);
5615                         si_enable_mgcg(rdev, false);
5616                 }
5617                 si_enable_gui_idle_interrupt(rdev, true);
5618         }
5619
5620         if (block & RADEON_CG_BLOCK_MC) {
5621                 si_enable_mc_mgcg(rdev, enable);
5622                 si_enable_mc_ls(rdev, enable);
5623         }
5624
5625         if (block & RADEON_CG_BLOCK_SDMA) {
5626                 si_enable_dma_mgcg(rdev, enable);
5627         }
5628
5629         if (block & RADEON_CG_BLOCK_BIF) {
5630                 si_enable_bif_mgls(rdev, enable);
5631         }
5632
5633         if (block & RADEON_CG_BLOCK_UVD) {
5634                 if (rdev->has_uvd) {
5635                         si_enable_uvd_mgcg(rdev, enable);
5636                 }
5637         }
5638
5639         if (block & RADEON_CG_BLOCK_HDP) {
5640                 si_enable_hdp_mgcg(rdev, enable);
5641                 si_enable_hdp_ls(rdev, enable);
5642         }
5643 }
5644
5645 static void si_init_cg(struct radeon_device *rdev)
5646 {
5647         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5648                             RADEON_CG_BLOCK_MC |
5649                             RADEON_CG_BLOCK_SDMA |
5650                             RADEON_CG_BLOCK_BIF |
5651                             RADEON_CG_BLOCK_HDP), true);
5652         if (rdev->has_uvd) {
5653                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5654                 si_init_uvd_internal_cg(rdev);
5655         }
5656 }
5657
5658 static void si_fini_cg(struct radeon_device *rdev)
5659 {
5660         if (rdev->has_uvd) {
5661                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5662         }
5663         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5664                             RADEON_CG_BLOCK_MC |
5665                             RADEON_CG_BLOCK_SDMA |
5666                             RADEON_CG_BLOCK_BIF |
5667                             RADEON_CG_BLOCK_HDP), false);
5668 }
5669
5670 u32 si_get_csb_size(struct radeon_device *rdev)
5671 {
5672         u32 count = 0;
5673         const struct cs_section_def *sect = NULL;
5674         const struct cs_extent_def *ext = NULL;
5675
5676         if (rdev->rlc.cs_data == NULL)
5677                 return 0;
5678
5679         /* begin clear state */
5680         count += 2;
5681         /* context control state */
5682         count += 3;
5683
5684         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5685                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5686                         if (sect->id == SECT_CONTEXT)
5687                                 count += 2 + ext->reg_count;
5688                         else
5689                                 return 0;
5690                 }
5691         }
5692         /* pa_sc_raster_config */
5693         count += 3;
5694         /* end clear state */
5695         count += 2;
5696         /* clear state */
5697         count += 2;
5698
5699         return count;
5700 }
5701
5702 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5703 {
5704         u32 count = 0, i;
5705         const struct cs_section_def *sect = NULL;
5706         const struct cs_extent_def *ext = NULL;
5707
5708         if (rdev->rlc.cs_data == NULL)
5709                 return;
5710         if (buffer == NULL)
5711                 return;
5712
5713         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5714         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5715
5716         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5717         buffer[count++] = cpu_to_le32(0x80000000);
5718         buffer[count++] = cpu_to_le32(0x80000000);
5719
5720         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5721                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5722                         if (sect->id == SECT_CONTEXT) {
5723                                 buffer[count++] =
5724                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5725                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5726                                 for (i = 0; i < ext->reg_count; i++)
5727                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
5728                         } else {
5729                                 return;
5730                         }
5731                 }
5732         }
5733
5734         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5735         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5736         switch (rdev->family) {
5737         case CHIP_TAHITI:
5738         case CHIP_PITCAIRN:
5739                 buffer[count++] = cpu_to_le32(0x2a00126a);
5740                 break;
5741         case CHIP_VERDE:
5742                 buffer[count++] = cpu_to_le32(0x0000124a);
5743                 break;
5744         case CHIP_OLAND:
5745                 buffer[count++] = cpu_to_le32(0x00000082);
5746                 break;
5747         case CHIP_HAINAN:
5748                 buffer[count++] = cpu_to_le32(0x00000000);
5749                 break;
5750         default:
5751                 buffer[count++] = cpu_to_le32(0x00000000);
5752                 break;
5753         }
5754
5755         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5756         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5757
5758         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5759         buffer[count++] = cpu_to_le32(0);
5760 }
5761
5762 static void si_init_pg(struct radeon_device *rdev)
5763 {
5764         if (rdev->pg_flags) {
5765                 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5766                         si_init_dma_pg(rdev);
5767                 }
5768                 si_init_ao_cu_mask(rdev);
5769                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5770                         si_init_gfx_cgpg(rdev);
5771                 } else {
5772                         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5773                         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5774                 }
5775                 si_enable_dma_pg(rdev, true);
5776                 si_enable_gfx_cgpg(rdev, true);
5777         } else {
5778                 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5779                 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5780         }
5781 }
5782
5783 static void si_fini_pg(struct radeon_device *rdev)
5784 {
5785         if (rdev->pg_flags) {
5786                 si_enable_dma_pg(rdev, false);
5787                 si_enable_gfx_cgpg(rdev, false);
5788         }
5789 }
5790
5791 /*
5792  * RLC
5793  */
5794 void si_rlc_reset(struct radeon_device *rdev)
5795 {
5796         u32 tmp = RREG32(GRBM_SOFT_RESET);
5797
5798         tmp |= SOFT_RESET_RLC;
5799         WREG32(GRBM_SOFT_RESET, tmp);
5800         udelay(50);
5801         tmp &= ~SOFT_RESET_RLC;
5802         WREG32(GRBM_SOFT_RESET, tmp);
5803         udelay(50);
5804 }
5805
5806 static void si_rlc_stop(struct radeon_device *rdev)
5807 {
5808         WREG32(RLC_CNTL, 0);
5809
5810         si_enable_gui_idle_interrupt(rdev, false);
5811
5812         si_wait_for_rlc_serdes(rdev);
5813 }
5814
5815 static void si_rlc_start(struct radeon_device *rdev)
5816 {
5817         WREG32(RLC_CNTL, RLC_ENABLE);
5818
5819         si_enable_gui_idle_interrupt(rdev, true);
5820
5821         udelay(50);
5822 }
5823
5824 static bool si_lbpw_supported(struct radeon_device *rdev)
5825 {
5826         u32 tmp;
5827
5828         /* Enable LBPW only for DDR3 */
5829         tmp = RREG32(MC_SEQ_MISC0);
5830         if ((tmp & 0xF0000000) == 0xB0000000)
5831                 return true;
5832         return false;
5833 }
5834
5835 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5836 {
5837         u32 tmp;
5838
5839         tmp = RREG32(RLC_LB_CNTL);
5840         if (enable)
5841                 tmp |= LOAD_BALANCE_ENABLE;
5842         else
5843                 tmp &= ~LOAD_BALANCE_ENABLE;
5844         WREG32(RLC_LB_CNTL, tmp);
5845
5846         if (!enable) {
5847                 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5848                 WREG32(SPI_LB_CU_MASK, 0x00ff);
5849         }
5850 }
5851
5852 static int si_rlc_resume(struct radeon_device *rdev)
5853 {
5854         u32 i;
5855
5856         if (!rdev->rlc_fw)
5857                 return -EINVAL;
5858
5859         si_rlc_stop(rdev);
5860
5861         si_rlc_reset(rdev);
5862
5863         si_init_pg(rdev);
5864
5865         si_init_cg(rdev);
5866
5867         WREG32(RLC_RL_BASE, 0);
5868         WREG32(RLC_RL_SIZE, 0);
5869         WREG32(RLC_LB_CNTL, 0);
5870         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5871         WREG32(RLC_LB_CNTR_INIT, 0);
5872         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5873
5874         WREG32(RLC_MC_CNTL, 0);
5875         WREG32(RLC_UCODE_CNTL, 0);
5876
5877         if (rdev->new_fw) {
5878                 const struct rlc_firmware_header_v1_0 *hdr =
5879                         (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5880                 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5881                 const __le32 *fw_data = (const __le32 *)
5882                         (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5883
5884                 radeon_ucode_print_rlc_hdr(&hdr->header);
5885
5886                 for (i = 0; i < fw_size; i++) {
5887                         WREG32(RLC_UCODE_ADDR, i);
5888                         WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5889                 }
5890         } else {
5891                 const __be32 *fw_data =
5892                         (const __be32 *)rdev->rlc_fw->data;
5893                 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5894                         WREG32(RLC_UCODE_ADDR, i);
5895                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5896                 }
5897         }
5898         WREG32(RLC_UCODE_ADDR, 0);
5899
5900         si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5901
5902         si_rlc_start(rdev);
5903
5904         return 0;
5905 }
5906
5907 static void si_enable_interrupts(struct radeon_device *rdev)
5908 {
5909         u32 ih_cntl = RREG32(IH_CNTL);
5910         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5911
5912         ih_cntl |= ENABLE_INTR;
5913         ih_rb_cntl |= IH_RB_ENABLE;
5914         WREG32(IH_CNTL, ih_cntl);
5915         WREG32(IH_RB_CNTL, ih_rb_cntl);
5916         rdev->ih.enabled = true;
5917 }
5918
5919 static void si_disable_interrupts(struct radeon_device *rdev)
5920 {
5921         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5922         u32 ih_cntl = RREG32(IH_CNTL);
5923
5924         ih_rb_cntl &= ~IH_RB_ENABLE;
5925         ih_cntl &= ~ENABLE_INTR;
5926         WREG32(IH_RB_CNTL, ih_rb_cntl);
5927         WREG32(IH_CNTL, ih_cntl);
5928         /* set rptr, wptr to 0 */
5929         WREG32(IH_RB_RPTR, 0);
5930         WREG32(IH_RB_WPTR, 0);
5931         rdev->ih.enabled = false;
5932         rdev->ih.rptr = 0;
5933 }
5934
5935 static void si_disable_interrupt_state(struct radeon_device *rdev)
5936 {
5937         u32 tmp;
5938
5939         tmp = RREG32(CP_INT_CNTL_RING0) &
5940                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5941         WREG32(CP_INT_CNTL_RING0, tmp);
5942         WREG32(CP_INT_CNTL_RING1, 0);
5943         WREG32(CP_INT_CNTL_RING2, 0);
5944         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5945         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5946         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5947         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5948         WREG32(GRBM_INT_CNTL, 0);
5949         WREG32(SRBM_INT_CNTL, 0);
5950         if (rdev->num_crtc >= 2) {
5951                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5952                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5953         }
5954         if (rdev->num_crtc >= 4) {
5955                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5956                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5957         }
5958         if (rdev->num_crtc >= 6) {
5959                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5960                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5961         }
5962
5963         if (rdev->num_crtc >= 2) {
5964                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5965                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5966         }
5967         if (rdev->num_crtc >= 4) {
5968                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5969                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5970         }
5971         if (rdev->num_crtc >= 6) {
5972                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5973                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5974         }
5975
5976         if (!ASIC_IS_NODCE(rdev)) {
5977                 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5978
5979                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5980                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5981                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5982                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5983                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5984                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5985                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5986                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5987                 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5988                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5989                 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5990                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5991         }
5992 }
5993
5994 static int si_irq_init(struct radeon_device *rdev)
5995 {
5996         int ret = 0;
5997         int rb_bufsz;
5998         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5999
6000         /* allocate ring */
6001         ret = r600_ih_ring_alloc(rdev);
6002         if (ret)
6003                 return ret;
6004
6005         /* disable irqs */
6006         si_disable_interrupts(rdev);
6007
6008         /* init rlc */
6009         ret = si_rlc_resume(rdev);
6010         if (ret) {
6011                 r600_ih_ring_fini(rdev);
6012                 return ret;
6013         }
6014
6015         /* setup interrupt control */
6016         /* set dummy read address to ring address */
6017         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6018         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6019         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6020          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6021          */
6022         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6023         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6024         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6025         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6026
6027         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6028         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6029
6030         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6031                       IH_WPTR_OVERFLOW_CLEAR |
6032                       (rb_bufsz << 1));
6033
6034         if (rdev->wb.enabled)
6035                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6036
6037         /* set the writeback address whether it's enabled or not */
6038         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6039         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6040
6041         WREG32(IH_RB_CNTL, ih_rb_cntl);
6042
6043         /* set rptr, wptr to 0 */
6044         WREG32(IH_RB_RPTR, 0);
6045         WREG32(IH_RB_WPTR, 0);
6046
6047         /* Default settings for IH_CNTL (disabled at first) */
6048         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6049         /* RPTR_REARM only works if msi's are enabled */
6050         if (rdev->msi_enabled)
6051                 ih_cntl |= RPTR_REARM;
6052         WREG32(IH_CNTL, ih_cntl);
6053
6054         /* force the active interrupt state to all disabled */
6055         si_disable_interrupt_state(rdev);
6056
6057         pci_set_master(rdev->pdev);
6058
6059         /* enable irqs */
6060         si_enable_interrupts(rdev);
6061
6062         return ret;
6063 }
6064
6065 int si_irq_set(struct radeon_device *rdev)
6066 {
6067         u32 cp_int_cntl;
6068         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6069         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6070         u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6071         u32 grbm_int_cntl = 0;
6072         u32 dma_cntl, dma_cntl1;
6073         u32 thermal_int = 0;
6074
6075         if (!rdev->irq.installed) {
6076                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6077                 return -EINVAL;
6078         }
6079         /* don't enable anything if the ih is disabled */
6080         if (!rdev->ih.enabled) {
6081                 si_disable_interrupts(rdev);
6082                 /* force the active interrupt state to all disabled */
6083                 si_disable_interrupt_state(rdev);
6084                 return 0;
6085         }
6086
6087         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6088                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6089
6090         if (!ASIC_IS_NODCE(rdev)) {
6091                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6092                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6093                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6094                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6095                 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6096                 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6097         }
6098
6099         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6100         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6101
6102         thermal_int = RREG32(CG_THERMAL_INT) &
6103                 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6104
6105         /* enable CP interrupts on all rings */
6106         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6107                 DRM_DEBUG("si_irq_set: sw int gfx\n");
6108                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6109         }
6110         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6111                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6112                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6113         }
6114         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6115                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6116                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6117         }
6118         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6119                 DRM_DEBUG("si_irq_set: sw int dma\n");
6120                 dma_cntl |= TRAP_ENABLE;
6121         }
6122
6123         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6124                 DRM_DEBUG("si_irq_set: sw int dma1\n");
6125                 dma_cntl1 |= TRAP_ENABLE;
6126         }
6127         if (rdev->irq.crtc_vblank_int[0] ||
6128             atomic_read(&rdev->irq.pflip[0])) {
6129                 DRM_DEBUG("si_irq_set: vblank 0\n");
6130                 crtc1 |= VBLANK_INT_MASK;
6131         }
6132         if (rdev->irq.crtc_vblank_int[1] ||
6133             atomic_read(&rdev->irq.pflip[1])) {
6134                 DRM_DEBUG("si_irq_set: vblank 1\n");
6135                 crtc2 |= VBLANK_INT_MASK;
6136         }
6137         if (rdev->irq.crtc_vblank_int[2] ||
6138             atomic_read(&rdev->irq.pflip[2])) {
6139                 DRM_DEBUG("si_irq_set: vblank 2\n");
6140                 crtc3 |= VBLANK_INT_MASK;
6141         }
6142         if (rdev->irq.crtc_vblank_int[3] ||
6143             atomic_read(&rdev->irq.pflip[3])) {
6144                 DRM_DEBUG("si_irq_set: vblank 3\n");
6145                 crtc4 |= VBLANK_INT_MASK;
6146         }
6147         if (rdev->irq.crtc_vblank_int[4] ||
6148             atomic_read(&rdev->irq.pflip[4])) {
6149                 DRM_DEBUG("si_irq_set: vblank 4\n");
6150                 crtc5 |= VBLANK_INT_MASK;
6151         }
6152         if (rdev->irq.crtc_vblank_int[5] ||
6153             atomic_read(&rdev->irq.pflip[5])) {
6154                 DRM_DEBUG("si_irq_set: vblank 5\n");
6155                 crtc6 |= VBLANK_INT_MASK;
6156         }
6157         if (rdev->irq.hpd[0]) {
6158                 DRM_DEBUG("si_irq_set: hpd 1\n");
6159                 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6160         }
6161         if (rdev->irq.hpd[1]) {
6162                 DRM_DEBUG("si_irq_set: hpd 2\n");
6163                 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6164         }
6165         if (rdev->irq.hpd[2]) {
6166                 DRM_DEBUG("si_irq_set: hpd 3\n");
6167                 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6168         }
6169         if (rdev->irq.hpd[3]) {
6170                 DRM_DEBUG("si_irq_set: hpd 4\n");
6171                 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6172         }
6173         if (rdev->irq.hpd[4]) {
6174                 DRM_DEBUG("si_irq_set: hpd 5\n");
6175                 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6176         }
6177         if (rdev->irq.hpd[5]) {
6178                 DRM_DEBUG("si_irq_set: hpd 6\n");
6179                 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6180         }
6181
6182         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6183         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6184         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6185
6186         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6187         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6188
6189         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6190
6191         if (rdev->irq.dpm_thermal) {
6192                 DRM_DEBUG("dpm thermal\n");
6193                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6194         }
6195
6196         if (rdev->num_crtc >= 2) {
6197                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6198                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6199         }
6200         if (rdev->num_crtc >= 4) {
6201                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6202                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6203         }
6204         if (rdev->num_crtc >= 6) {
6205                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6206                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6207         }
6208
6209         if (rdev->num_crtc >= 2) {
6210                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6211                        GRPH_PFLIP_INT_MASK);
6212                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6213                        GRPH_PFLIP_INT_MASK);
6214         }
6215         if (rdev->num_crtc >= 4) {
6216                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6217                        GRPH_PFLIP_INT_MASK);
6218                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6219                        GRPH_PFLIP_INT_MASK);
6220         }
6221         if (rdev->num_crtc >= 6) {
6222                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6223                        GRPH_PFLIP_INT_MASK);
6224                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6225                        GRPH_PFLIP_INT_MASK);
6226         }
6227
6228         if (!ASIC_IS_NODCE(rdev)) {
6229                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6230                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6231                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6232                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6233                 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6234                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6235         }
6236
6237         WREG32(CG_THERMAL_INT, thermal_int);
6238
6239         /* posting read */
6240         RREG32(SRBM_STATUS);
6241
6242         return 0;
6243 }
6244
6245 static inline void si_irq_ack(struct radeon_device *rdev)
6246 {
6247         u32 tmp;
6248
6249         if (ASIC_IS_NODCE(rdev))
6250                 return;
6251
6252         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6253         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6254         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6255         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6256         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6257         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6258         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6259         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6260         if (rdev->num_crtc >= 4) {
6261                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6262                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6263         }
6264         if (rdev->num_crtc >= 6) {
6265                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6266                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6267         }
6268
6269         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6270                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6271         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6272                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6273         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6274                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6275         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6276                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6277         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6278                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6279         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6280                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6281
6282         if (rdev->num_crtc >= 4) {
6283                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6284                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6285                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6286                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6287                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6288                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6289                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6290                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6291                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6292                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6293                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6294                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6295         }
6296
6297         if (rdev->num_crtc >= 6) {
6298                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6299                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6300                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6301                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6302                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6303                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6304                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6305                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6306                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6307                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6308                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6309                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6310         }
6311
6312         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6313                 tmp = RREG32(DC_HPD1_INT_CONTROL);
6314                 tmp |= DC_HPDx_INT_ACK;
6315                 WREG32(DC_HPD1_INT_CONTROL, tmp);
6316         }
6317         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6318                 tmp = RREG32(DC_HPD2_INT_CONTROL);
6319                 tmp |= DC_HPDx_INT_ACK;
6320                 WREG32(DC_HPD2_INT_CONTROL, tmp);
6321         }
6322         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6323                 tmp = RREG32(DC_HPD3_INT_CONTROL);
6324                 tmp |= DC_HPDx_INT_ACK;
6325                 WREG32(DC_HPD3_INT_CONTROL, tmp);
6326         }
6327         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6328                 tmp = RREG32(DC_HPD4_INT_CONTROL);
6329                 tmp |= DC_HPDx_INT_ACK;
6330                 WREG32(DC_HPD4_INT_CONTROL, tmp);
6331         }
6332         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6333                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6334                 tmp |= DC_HPDx_INT_ACK;
6335                 WREG32(DC_HPD5_INT_CONTROL, tmp);
6336         }
6337         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6338                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6339                 tmp |= DC_HPDx_INT_ACK;
6340                 WREG32(DC_HPD6_INT_CONTROL, tmp);
6341         }
6342
6343         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6344                 tmp = RREG32(DC_HPD1_INT_CONTROL);
6345                 tmp |= DC_HPDx_RX_INT_ACK;
6346                 WREG32(DC_HPD1_INT_CONTROL, tmp);
6347         }
6348         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6349                 tmp = RREG32(DC_HPD2_INT_CONTROL);
6350                 tmp |= DC_HPDx_RX_INT_ACK;
6351                 WREG32(DC_HPD2_INT_CONTROL, tmp);
6352         }
6353         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6354                 tmp = RREG32(DC_HPD3_INT_CONTROL);
6355                 tmp |= DC_HPDx_RX_INT_ACK;
6356                 WREG32(DC_HPD3_INT_CONTROL, tmp);
6357         }
6358         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6359                 tmp = RREG32(DC_HPD4_INT_CONTROL);
6360                 tmp |= DC_HPDx_RX_INT_ACK;
6361                 WREG32(DC_HPD4_INT_CONTROL, tmp);
6362         }
6363         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6364                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6365                 tmp |= DC_HPDx_RX_INT_ACK;
6366                 WREG32(DC_HPD5_INT_CONTROL, tmp);
6367         }
6368         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6369                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6370                 tmp |= DC_HPDx_RX_INT_ACK;
6371                 WREG32(DC_HPD6_INT_CONTROL, tmp);
6372         }
6373 }
6374
6375 static void si_irq_disable(struct radeon_device *rdev)
6376 {
6377         si_disable_interrupts(rdev);
6378         /* Wait and acknowledge irq */
6379         mdelay(1);
6380         si_irq_ack(rdev);
6381         si_disable_interrupt_state(rdev);
6382 }
6383
6384 static void si_irq_suspend(struct radeon_device *rdev)
6385 {
6386         si_irq_disable(rdev);
6387         si_rlc_stop(rdev);
6388 }
6389
6390 static void si_irq_fini(struct radeon_device *rdev)
6391 {
6392         si_irq_suspend(rdev);
6393         r600_ih_ring_fini(rdev);
6394 }
6395
6396 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6397 {
6398         u32 wptr, tmp;
6399
6400         if (rdev->wb.enabled)
6401                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6402         else
6403                 wptr = RREG32(IH_RB_WPTR);
6404
6405         if (wptr & RB_OVERFLOW) {
6406                 wptr &= ~RB_OVERFLOW;
6407                 /* When a ring buffer overflow happen start parsing interrupt
6408                  * from the last not overwritten vector (wptr + 16). Hopefully
6409                  * this should allow us to catchup.
6410                  */
6411                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6412                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6413                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6414                 tmp = RREG32(IH_RB_CNTL);
6415                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6416                 WREG32(IH_RB_CNTL, tmp);
6417         }
6418         return (wptr & rdev->ih.ptr_mask);
6419 }
6420
6421 /*        SI IV Ring
6422  * Each IV ring entry is 128 bits:
6423  * [7:0]    - interrupt source id
6424  * [31:8]   - reserved
6425  * [59:32]  - interrupt source data
6426  * [63:60]  - reserved
6427  * [71:64]  - RINGID
6428  * [79:72]  - VMID
6429  * [127:80] - reserved
6430  */
6431 int si_irq_process(struct radeon_device *rdev)
6432 {
6433         u32 wptr;
6434         u32 rptr;
6435         u32 src_id, src_data, ring_id;
6436         u32 ring_index;
6437         bool queue_hotplug = false;
6438         bool queue_dp = false;
6439         bool queue_thermal = false;
6440         u32 status, addr;
6441
6442         if (!rdev->ih.enabled || rdev->shutdown)
6443                 return IRQ_NONE;
6444
6445         wptr = si_get_ih_wptr(rdev);
6446
6447 restart_ih:
6448         /* is somebody else already processing irqs? */
6449         if (atomic_xchg(&rdev->ih.lock, 1))
6450                 return IRQ_NONE;
6451
6452         rptr = rdev->ih.rptr;
6453         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6454
6455         /* Order reading of wptr vs. reading of IH ring data */
6456         rmb();
6457
6458         /* display interrupts */
6459         si_irq_ack(rdev);
6460
6461         while (rptr != wptr) {
6462                 /* wptr/rptr are in bytes! */
6463                 ring_index = rptr / 4;
6464                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6465                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6466                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6467
6468                 switch (src_id) {
6469                 case 1: /* D1 vblank/vline */
6470                         switch (src_data) {
6471                         case 0: /* D1 vblank */
6472                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6473                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6474
6475                                 if (rdev->irq.crtc_vblank_int[0]) {
6476                                         drm_handle_vblank(rdev->ddev, 0);
6477                                         rdev->pm.vblank_sync = true;
6478                                         wake_up(&rdev->irq.vblank_queue);
6479                                 }
6480                                 if (atomic_read(&rdev->irq.pflip[0]))
6481                                         radeon_crtc_handle_vblank(rdev, 0);
6482                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6483                                 DRM_DEBUG("IH: D1 vblank\n");
6484
6485                                 break;
6486                         case 1: /* D1 vline */
6487                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6488                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6489
6490                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6491                                 DRM_DEBUG("IH: D1 vline\n");
6492
6493                                 break;
6494                         default:
6495                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6496                                 break;
6497                         }
6498                         break;
6499                 case 2: /* D2 vblank/vline */
6500                         switch (src_data) {
6501                         case 0: /* D2 vblank */
6502                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6503                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6504
6505                                 if (rdev->irq.crtc_vblank_int[1]) {
6506                                         drm_handle_vblank(rdev->ddev, 1);
6507                                         rdev->pm.vblank_sync = true;
6508                                         wake_up(&rdev->irq.vblank_queue);
6509                                 }
6510                                 if (atomic_read(&rdev->irq.pflip[1]))
6511                                         radeon_crtc_handle_vblank(rdev, 1);
6512                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6513                                 DRM_DEBUG("IH: D2 vblank\n");
6514
6515                                 break;
6516                         case 1: /* D2 vline */
6517                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6518                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6519
6520                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6521                                 DRM_DEBUG("IH: D2 vline\n");
6522
6523                                 break;
6524                         default:
6525                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6526                                 break;
6527                         }
6528                         break;
6529                 case 3: /* D3 vblank/vline */
6530                         switch (src_data) {
6531                         case 0: /* D3 vblank */
6532                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6533                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6534
6535                                 if (rdev->irq.crtc_vblank_int[2]) {
6536                                         drm_handle_vblank(rdev->ddev, 2);
6537                                         rdev->pm.vblank_sync = true;
6538                                         wake_up(&rdev->irq.vblank_queue);
6539                                 }
6540                                 if (atomic_read(&rdev->irq.pflip[2]))
6541                                         radeon_crtc_handle_vblank(rdev, 2);
6542                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6543                                 DRM_DEBUG("IH: D3 vblank\n");
6544
6545                                 break;
6546                         case 1: /* D3 vline */
6547                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6548                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6549
6550                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6551                                 DRM_DEBUG("IH: D3 vline\n");
6552
6553                                 break;
6554                         default:
6555                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6556                                 break;
6557                         }
6558                         break;
6559                 case 4: /* D4 vblank/vline */
6560                         switch (src_data) {
6561                         case 0: /* D4 vblank */
6562                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6563                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6564
6565                                 if (rdev->irq.crtc_vblank_int[3]) {
6566                                         drm_handle_vblank(rdev->ddev, 3);
6567                                         rdev->pm.vblank_sync = true;
6568                                         wake_up(&rdev->irq.vblank_queue);
6569                                 }
6570                                 if (atomic_read(&rdev->irq.pflip[3]))
6571                                         radeon_crtc_handle_vblank(rdev, 3);
6572                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6573                                 DRM_DEBUG("IH: D4 vblank\n");
6574
6575                                 break;
6576                         case 1: /* D4 vline */
6577                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6578                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6579
6580                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6581                                 DRM_DEBUG("IH: D4 vline\n");
6582
6583                                 break;
6584                         default:
6585                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6586                                 break;
6587                         }
6588                         break;
6589                 case 5: /* D5 vblank/vline */
6590                         switch (src_data) {
6591                         case 0: /* D5 vblank */
6592                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6593                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6594
6595                                 if (rdev->irq.crtc_vblank_int[4]) {
6596                                         drm_handle_vblank(rdev->ddev, 4);
6597                                         rdev->pm.vblank_sync = true;
6598                                         wake_up(&rdev->irq.vblank_queue);
6599                                 }
6600                                 if (atomic_read(&rdev->irq.pflip[4]))
6601                                         radeon_crtc_handle_vblank(rdev, 4);
6602                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6603                                 DRM_DEBUG("IH: D5 vblank\n");
6604
6605                                 break;
6606                         case 1: /* D5 vline */
6607                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6608                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6609
6610                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6611                                 DRM_DEBUG("IH: D5 vline\n");
6612
6613                                 break;
6614                         default:
6615                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6616                                 break;
6617                         }
6618                         break;
6619                 case 6: /* D6 vblank/vline */
6620                         switch (src_data) {
6621                         case 0: /* D6 vblank */
6622                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6623                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6624
6625                                 if (rdev->irq.crtc_vblank_int[5]) {
6626                                         drm_handle_vblank(rdev->ddev, 5);
6627                                         rdev->pm.vblank_sync = true;
6628                                         wake_up(&rdev->irq.vblank_queue);
6629                                 }
6630                                 if (atomic_read(&rdev->irq.pflip[5]))
6631                                         radeon_crtc_handle_vblank(rdev, 5);
6632                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6633                                 DRM_DEBUG("IH: D6 vblank\n");
6634
6635                                 break;
6636                         case 1: /* D6 vline */
6637                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6638                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6639
6640                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6641                                 DRM_DEBUG("IH: D6 vline\n");
6642
6643                                 break;
6644                         default:
6645                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6646                                 break;
6647                         }
6648                         break;
6649                 case 8: /* D1 page flip */
6650                 case 10: /* D2 page flip */
6651                 case 12: /* D3 page flip */
6652                 case 14: /* D4 page flip */
6653                 case 16: /* D5 page flip */
6654                 case 18: /* D6 page flip */
6655                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6656                         if (radeon_use_pflipirq > 0)
6657                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6658                         break;
6659                 case 42: /* HPD hotplug */
6660                         switch (src_data) {
6661                         case 0:
6662                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6663                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6664
6665                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6666                                 queue_hotplug = true;
6667                                 DRM_DEBUG("IH: HPD1\n");
6668
6669                                 break;
6670                         case 1:
6671                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6672                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6673
6674                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6675                                 queue_hotplug = true;
6676                                 DRM_DEBUG("IH: HPD2\n");
6677
6678                                 break;
6679                         case 2:
6680                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6681                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6682
6683                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6684                                 queue_hotplug = true;
6685                                 DRM_DEBUG("IH: HPD3\n");
6686
6687                                 break;
6688                         case 3:
6689                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6690                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6691
6692                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6693                                 queue_hotplug = true;
6694                                 DRM_DEBUG("IH: HPD4\n");
6695
6696                                 break;
6697                         case 4:
6698                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6699                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6700
6701                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6702                                 queue_hotplug = true;
6703                                 DRM_DEBUG("IH: HPD5\n");
6704
6705                                 break;
6706                         case 5:
6707                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6708                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6709
6710                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6711                                 queue_hotplug = true;
6712                                 DRM_DEBUG("IH: HPD6\n");
6713
6714                                 break;
6715                         case 6:
6716                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6717                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6718
6719                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6720                                 queue_dp = true;
6721                                 DRM_DEBUG("IH: HPD_RX 1\n");
6722
6723                                 break;
6724                         case 7:
6725                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6726                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6727
6728                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6729                                 queue_dp = true;
6730                                 DRM_DEBUG("IH: HPD_RX 2\n");
6731
6732                                 break;
6733                         case 8:
6734                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6735                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6736
6737                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6738                                 queue_dp = true;
6739                                 DRM_DEBUG("IH: HPD_RX 3\n");
6740
6741                                 break;
6742                         case 9:
6743                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6744                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6745
6746                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6747                                 queue_dp = true;
6748                                 DRM_DEBUG("IH: HPD_RX 4\n");
6749
6750                                 break;
6751                         case 10:
6752                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6753                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6754
6755                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6756                                 queue_dp = true;
6757                                 DRM_DEBUG("IH: HPD_RX 5\n");
6758
6759                                 break;
6760                         case 11:
6761                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6762                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6763
6764                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6765                                 queue_dp = true;
6766                                 DRM_DEBUG("IH: HPD_RX 6\n");
6767
6768                                 break;
6769                         default:
6770                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6771                                 break;
6772                         }
6773                         break;
6774                 case 96:
6775                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6776                         WREG32(SRBM_INT_ACK, 0x1);
6777                         break;
6778                 case 124: /* UVD */
6779                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6780                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6781                         break;
6782                 case 146:
6783                 case 147:
6784                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6785                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6786                         /* reset addr and status */
6787                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6788                         if (addr == 0x0 && status == 0x0)
6789                                 break;
6790                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6791                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6792                                 addr);
6793                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6794                                 status);
6795                         si_vm_decode_fault(rdev, status, addr);
6796                         break;
6797                 case 176: /* RINGID0 CP_INT */
6798                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6799                         break;
6800                 case 177: /* RINGID1 CP_INT */
6801                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6802                         break;
6803                 case 178: /* RINGID2 CP_INT */
6804                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6805                         break;
6806                 case 181: /* CP EOP event */
6807                         DRM_DEBUG("IH: CP EOP\n");
6808                         switch (ring_id) {
6809                         case 0:
6810                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6811                                 break;
6812                         case 1:
6813                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6814                                 break;
6815                         case 2:
6816                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6817                                 break;
6818                         }
6819                         break;
6820                 case 224: /* DMA trap event */
6821                         DRM_DEBUG("IH: DMA trap\n");
6822                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6823                         break;
6824                 case 230: /* thermal low to high */
6825                         DRM_DEBUG("IH: thermal low to high\n");
6826                         rdev->pm.dpm.thermal.high_to_low = false;
6827                         queue_thermal = true;
6828                         break;
6829                 case 231: /* thermal high to low */
6830                         DRM_DEBUG("IH: thermal high to low\n");
6831                         rdev->pm.dpm.thermal.high_to_low = true;
6832                         queue_thermal = true;
6833                         break;
6834                 case 233: /* GUI IDLE */
6835                         DRM_DEBUG("IH: GUI idle\n");
6836                         break;
6837                 case 244: /* DMA trap event */
6838                         DRM_DEBUG("IH: DMA1 trap\n");
6839                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6840                         break;
6841                 default:
6842                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6843                         break;
6844                 }
6845
6846                 /* wptr/rptr are in bytes! */
6847                 rptr += 16;
6848                 rptr &= rdev->ih.ptr_mask;
6849                 WREG32(IH_RB_RPTR, rptr);
6850         }
6851         if (queue_dp)
6852                 schedule_work(&rdev->dp_work);
6853         if (queue_hotplug)
6854                 schedule_delayed_work(&rdev->hotplug_work, 0);
6855         if (queue_thermal && rdev->pm.dpm_enabled)
6856                 schedule_work(&rdev->pm.dpm.thermal.work);
6857         rdev->ih.rptr = rptr;
6858         atomic_set(&rdev->ih.lock, 0);
6859
6860         /* make sure wptr hasn't changed while processing */
6861         wptr = si_get_ih_wptr(rdev);
6862         if (wptr != rptr)
6863                 goto restart_ih;
6864
6865         return IRQ_HANDLED;
6866 }
6867
6868 /*
6869  * startup/shutdown callbacks
6870  */
6871 static int si_startup(struct radeon_device *rdev)
6872 {
6873         struct radeon_ring *ring;
6874         int r;
6875
6876         /* enable pcie gen2/3 link */
6877         si_pcie_gen3_enable(rdev);
6878         /* enable aspm */
6879         si_program_aspm(rdev);
6880
6881         /* scratch needs to be initialized before MC */
6882         r = r600_vram_scratch_init(rdev);
6883         if (r)
6884                 return r;
6885
6886         si_mc_program(rdev);
6887
6888         if (!rdev->pm.dpm_enabled) {
6889                 r = si_mc_load_microcode(rdev);
6890                 if (r) {
6891                         DRM_ERROR("Failed to load MC firmware!\n");
6892                         return r;
6893                 }
6894         }
6895
6896         r = si_pcie_gart_enable(rdev);
6897         if (r)
6898                 return r;
6899         si_gpu_init(rdev);
6900
6901         /* allocate rlc buffers */
6902         if (rdev->family == CHIP_VERDE) {
6903                 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6904                 rdev->rlc.reg_list_size =
6905                         (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6906         }
6907         rdev->rlc.cs_data = si_cs_data;
6908         r = sumo_rlc_init(rdev);
6909         if (r) {
6910                 DRM_ERROR("Failed to init rlc BOs!\n");
6911                 return r;
6912         }
6913
6914         /* allocate wb buffer */
6915         r = radeon_wb_init(rdev);
6916         if (r)
6917                 return r;
6918
6919         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6920         if (r) {
6921                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6922                 return r;
6923         }
6924
6925         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6926         if (r) {
6927                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6928                 return r;
6929         }
6930
6931         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6932         if (r) {
6933                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6934                 return r;
6935         }
6936
6937         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6938         if (r) {
6939                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6940                 return r;
6941         }
6942
6943         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6944         if (r) {
6945                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6946                 return r;
6947         }
6948
6949         if (rdev->has_uvd) {
6950                 r = uvd_v2_2_resume(rdev);
6951                 if (!r) {
6952                         r = radeon_fence_driver_start_ring(rdev,
6953                                                            R600_RING_TYPE_UVD_INDEX);
6954                         if (r)
6955                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6956                 }
6957                 if (r)
6958                         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6959         }
6960
6961         r = radeon_vce_resume(rdev);
6962         if (!r) {
6963                 r = vce_v1_0_resume(rdev);
6964                 if (!r)
6965                         r = radeon_fence_driver_start_ring(rdev,
6966                                                            TN_RING_TYPE_VCE1_INDEX);
6967                 if (!r)
6968                         r = radeon_fence_driver_start_ring(rdev,
6969                                                            TN_RING_TYPE_VCE2_INDEX);
6970         }
6971         if (r) {
6972                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
6973                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6974                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6975         }
6976
6977         /* Enable IRQ */
6978         if (!rdev->irq.installed) {
6979                 r = radeon_irq_kms_init(rdev);
6980                 if (r)
6981                         return r;
6982         }
6983
6984         r = si_irq_init(rdev);
6985         if (r) {
6986                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6987                 radeon_irq_kms_fini(rdev);
6988                 return r;
6989         }
6990         si_irq_set(rdev);
6991
6992         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6993         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6994                              RADEON_CP_PACKET2);
6995         if (r)
6996                 return r;
6997
6998         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6999         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7000                              RADEON_CP_PACKET2);
7001         if (r)
7002                 return r;
7003
7004         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7005         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7006                              RADEON_CP_PACKET2);
7007         if (r)
7008                 return r;
7009
7010         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7011         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7012                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7013         if (r)
7014                 return r;
7015
7016         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7017         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7018                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7019         if (r)
7020                 return r;
7021
7022         r = si_cp_load_microcode(rdev);
7023         if (r)
7024                 return r;
7025         r = si_cp_resume(rdev);
7026         if (r)
7027                 return r;
7028
7029         r = cayman_dma_resume(rdev);
7030         if (r)
7031                 return r;
7032
7033         if (rdev->has_uvd) {
7034                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7035                 if (ring->ring_size) {
7036                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7037                                              RADEON_CP_PACKET2);
7038                         if (!r)
7039                                 r = uvd_v1_0_init(rdev);
7040                         if (r)
7041                                 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7042                 }
7043         }
7044
7045         r = -ENOENT;
7046
7047         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7048         if (ring->ring_size)
7049                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7050                                      VCE_CMD_NO_OP);
7051
7052         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7053         if (ring->ring_size)
7054                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7055                                      VCE_CMD_NO_OP);
7056
7057         if (!r)
7058                 r = vce_v1_0_init(rdev);
7059         else if (r != -ENOENT)
7060                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7061
7062         r = radeon_ib_pool_init(rdev);
7063         if (r) {
7064                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7065                 return r;
7066         }
7067
7068         r = radeon_vm_manager_init(rdev);
7069         if (r) {
7070                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7071                 return r;
7072         }
7073
7074         r = radeon_audio_init(rdev);
7075         if (r)
7076                 return r;
7077
7078         return 0;
7079 }
7080
7081 int si_resume(struct radeon_device *rdev)
7082 {
7083         int r;
7084
7085         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7086          * posting will perform necessary task to bring back GPU into good
7087          * shape.
7088          */
7089         /* post card */
7090         atom_asic_init(rdev->mode_info.atom_context);
7091
7092         /* init golden registers */
7093         si_init_golden_registers(rdev);
7094
7095         if (rdev->pm.pm_method == PM_METHOD_DPM)
7096                 radeon_pm_resume(rdev);
7097
7098         rdev->accel_working = true;
7099         r = si_startup(rdev);
7100         if (r) {
7101                 DRM_ERROR("si startup failed on resume\n");
7102                 rdev->accel_working = false;
7103                 return r;
7104         }
7105
7106         return r;
7107
7108 }
7109
7110 int si_suspend(struct radeon_device *rdev)
7111 {
7112         radeon_pm_suspend(rdev);
7113         radeon_audio_fini(rdev);
7114         radeon_vm_manager_fini(rdev);
7115         si_cp_enable(rdev, false);
7116         cayman_dma_stop(rdev);
7117         if (rdev->has_uvd) {
7118                 uvd_v1_0_fini(rdev);
7119                 radeon_uvd_suspend(rdev);
7120                 radeon_vce_suspend(rdev);
7121         }
7122         si_fini_pg(rdev);
7123         si_fini_cg(rdev);
7124         si_irq_suspend(rdev);
7125         radeon_wb_disable(rdev);
7126         si_pcie_gart_disable(rdev);
7127         return 0;
7128 }
7129
7130 /* Plan is to move initialization in that function and use
7131  * helper function so that radeon_device_init pretty much
7132  * do nothing more than calling asic specific function. This
7133  * should also allow to remove a bunch of callback function
7134  * like vram_info.
7135  */
7136 int si_init(struct radeon_device *rdev)
7137 {
7138         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7139         int r;
7140
7141         /* Read BIOS */
7142         if (!radeon_get_bios(rdev)) {
7143                 if (ASIC_IS_AVIVO(rdev))
7144                         return -EINVAL;
7145         }
7146         /* Must be an ATOMBIOS */
7147         if (!rdev->is_atom_bios) {
7148                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7149                 return -EINVAL;
7150         }
7151         r = radeon_atombios_init(rdev);
7152         if (r)
7153                 return r;
7154
7155         /* Post card if necessary */
7156         if (!radeon_card_posted(rdev)) {
7157                 if (!rdev->bios) {
7158                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7159                         return -EINVAL;
7160                 }
7161                 DRM_INFO("GPU not posted. posting now...\n");
7162                 atom_asic_init(rdev->mode_info.atom_context);
7163         }
7164         /* init golden registers */
7165         si_init_golden_registers(rdev);
7166         /* Initialize scratch registers */
7167         si_scratch_init(rdev);
7168         /* Initialize surface registers */
7169         radeon_surface_init(rdev);
7170         /* Initialize clocks */
7171         radeon_get_clock_info(rdev->ddev);
7172
7173         /* Fence driver */
7174         r = radeon_fence_driver_init(rdev);
7175         if (r)
7176                 return r;
7177
7178         /* initialize memory controller */
7179         r = si_mc_init(rdev);
7180         if (r)
7181                 return r;
7182         /* Memory manager */
7183         r = radeon_bo_init(rdev);
7184         if (r)
7185                 return r;
7186
7187         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7188             !rdev->rlc_fw || !rdev->mc_fw) {
7189                 r = si_init_microcode(rdev);
7190                 if (r) {
7191                         DRM_ERROR("Failed to load firmware!\n");
7192                         return r;
7193                 }
7194         }
7195
7196         /* Initialize power management */
7197         radeon_pm_init(rdev);
7198
7199         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7200         ring->ring_obj = NULL;
7201         r600_ring_init(rdev, ring, 1024 * 1024);
7202
7203         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7204         ring->ring_obj = NULL;
7205         r600_ring_init(rdev, ring, 1024 * 1024);
7206
7207         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7208         ring->ring_obj = NULL;
7209         r600_ring_init(rdev, ring, 1024 * 1024);
7210
7211         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7212         ring->ring_obj = NULL;
7213         r600_ring_init(rdev, ring, 64 * 1024);
7214
7215         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7216         ring->ring_obj = NULL;
7217         r600_ring_init(rdev, ring, 64 * 1024);
7218
7219         if (rdev->has_uvd) {
7220                 r = radeon_uvd_init(rdev);
7221                 if (!r) {
7222                         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7223                         ring->ring_obj = NULL;
7224                         r600_ring_init(rdev, ring, 4096);
7225                 }
7226         }
7227
7228         r = radeon_vce_init(rdev);
7229         if (!r) {
7230                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7231                 ring->ring_obj = NULL;
7232                 r600_ring_init(rdev, ring, 4096);
7233
7234                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7235                 ring->ring_obj = NULL;
7236                 r600_ring_init(rdev, ring, 4096);
7237         }
7238
7239         rdev->ih.ring_obj = NULL;
7240         r600_ih_ring_init(rdev, 64 * 1024);
7241
7242         r = r600_pcie_gart_init(rdev);
7243         if (r)
7244                 return r;
7245
7246         rdev->accel_working = true;
7247         r = si_startup(rdev);
7248         if (r) {
7249                 dev_err(rdev->dev, "disabling GPU acceleration\n");
7250                 si_cp_fini(rdev);
7251                 cayman_dma_fini(rdev);
7252                 si_irq_fini(rdev);
7253                 sumo_rlc_fini(rdev);
7254                 radeon_wb_fini(rdev);
7255                 radeon_ib_pool_fini(rdev);
7256                 radeon_vm_manager_fini(rdev);
7257                 radeon_irq_kms_fini(rdev);
7258                 si_pcie_gart_fini(rdev);
7259                 rdev->accel_working = false;
7260         }
7261
7262         /* Don't start up if the MC ucode is missing.
7263          * The default clocks and voltages before the MC ucode
7264          * is loaded are not suffient for advanced operations.
7265          */
7266         if (!rdev->mc_fw) {
7267                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7268                 return -EINVAL;
7269         }
7270
7271         return 0;
7272 }
7273
7274 void si_fini(struct radeon_device *rdev)
7275 {
7276         radeon_pm_fini(rdev);
7277         si_cp_fini(rdev);
7278         cayman_dma_fini(rdev);
7279         si_fini_pg(rdev);
7280         si_fini_cg(rdev);
7281         si_irq_fini(rdev);
7282         sumo_rlc_fini(rdev);
7283         radeon_wb_fini(rdev);
7284         radeon_vm_manager_fini(rdev);
7285         radeon_ib_pool_fini(rdev);
7286         radeon_irq_kms_fini(rdev);
7287         if (rdev->has_uvd) {
7288                 uvd_v1_0_fini(rdev);
7289                 radeon_uvd_fini(rdev);
7290                 radeon_vce_fini(rdev);
7291         }
7292         si_pcie_gart_fini(rdev);
7293         r600_vram_scratch_fini(rdev);
7294         radeon_gem_fini(rdev);
7295         radeon_fence_driver_fini(rdev);
7296         radeon_bo_fini(rdev);
7297         radeon_atombios_fini(rdev);
7298         kfree(rdev->bios);
7299         rdev->bios = NULL;
7300 }
7301
7302 /**
7303  * si_get_gpu_clock_counter - return GPU clock counter snapshot
7304  *
7305  * @rdev: radeon_device pointer
7306  *
7307  * Fetches a GPU clock counter snapshot (SI).
7308  * Returns the 64 bit clock counter snapshot.
7309  */
7310 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7311 {
7312         uint64_t clock;
7313
7314         mutex_lock(&rdev->gpu_clock_mutex);
7315         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7316         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7317                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7318         mutex_unlock(&rdev->gpu_clock_mutex);
7319         return clock;
7320 }
7321
7322 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7323 {
7324         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7325         int r;
7326
7327         /* bypass vclk and dclk with bclk */
7328         WREG32_P(CG_UPLL_FUNC_CNTL_2,
7329                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7330                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7331
7332         /* put PLL in bypass mode */
7333         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7334
7335         if (!vclk || !dclk) {
7336                 /* keep the Bypass mode */
7337                 return 0;
7338         }
7339
7340         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7341                                           16384, 0x03FFFFFF, 0, 128, 5,
7342                                           &fb_div, &vclk_div, &dclk_div);
7343         if (r)
7344                 return r;
7345
7346         /* set RESET_ANTI_MUX to 0 */
7347         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7348
7349         /* set VCO_MODE to 1 */
7350         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7351
7352         /* disable sleep mode */
7353         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7354
7355         /* deassert UPLL_RESET */
7356         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7357
7358         mdelay(1);
7359
7360         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7361         if (r)
7362                 return r;
7363
7364         /* assert UPLL_RESET again */
7365         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7366
7367         /* disable spread spectrum. */
7368         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7369
7370         /* set feedback divider */
7371         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7372
7373         /* set ref divider to 0 */
7374         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7375
7376         if (fb_div < 307200)
7377                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7378         else
7379                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7380
7381         /* set PDIV_A and PDIV_B */
7382         WREG32_P(CG_UPLL_FUNC_CNTL_2,
7383                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7384                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7385
7386         /* give the PLL some time to settle */
7387         mdelay(15);
7388
7389         /* deassert PLL_RESET */
7390         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7391
7392         mdelay(15);
7393
7394         /* switch from bypass mode to normal mode */
7395         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7396
7397         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7398         if (r)
7399                 return r;
7400
7401         /* switch VCLK and DCLK selection */
7402         WREG32_P(CG_UPLL_FUNC_CNTL_2,
7403                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7404                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7405
7406         mdelay(100);
7407
7408         return 0;
7409 }
7410
7411 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7412 {
7413         struct pci_dev *root = rdev->pdev->bus->self;
7414         int bridge_pos, gpu_pos;
7415         u32 speed_cntl, mask, current_data_rate;
7416         int ret, i;
7417         u16 tmp16;
7418
7419         if (pci_is_root_bus(rdev->pdev->bus))
7420                 return;
7421
7422         if (radeon_pcie_gen2 == 0)
7423                 return;
7424
7425         if (rdev->flags & RADEON_IS_IGP)
7426                 return;
7427
7428         if (!(rdev->flags & RADEON_IS_PCIE))
7429                 return;
7430
7431         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7432         if (ret != 0)
7433                 return;
7434
7435         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7436                 return;
7437
7438         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7439         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7440                 LC_CURRENT_DATA_RATE_SHIFT;
7441         if (mask & DRM_PCIE_SPEED_80) {
7442                 if (current_data_rate == 2) {
7443                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7444                         return;
7445                 }
7446                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7447         } else if (mask & DRM_PCIE_SPEED_50) {
7448                 if (current_data_rate == 1) {
7449                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7450                         return;
7451                 }
7452                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7453         }
7454
7455         bridge_pos = pci_pcie_cap(root);
7456         if (!bridge_pos)
7457                 return;
7458
7459         gpu_pos = pci_pcie_cap(rdev->pdev);
7460         if (!gpu_pos)
7461                 return;
7462
7463         if (mask & DRM_PCIE_SPEED_80) {
7464                 /* re-try equalization if gen3 is not already enabled */
7465                 if (current_data_rate != 2) {
7466                         u16 bridge_cfg, gpu_cfg;
7467                         u16 bridge_cfg2, gpu_cfg2;
7468                         u32 max_lw, current_lw, tmp;
7469
7470                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7471                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7472
7473                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7474                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7475
7476                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7477                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7478
7479                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7480                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7481                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7482
7483                         if (current_lw < max_lw) {
7484                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7485                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7486                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7487                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7488                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7489                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7490                                 }
7491                         }
7492
7493                         for (i = 0; i < 10; i++) {
7494                                 /* check status */
7495                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7496                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7497                                         break;
7498
7499                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7500                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7501
7502                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7503                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7504
7505                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7506                                 tmp |= LC_SET_QUIESCE;
7507                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7508
7509                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7510                                 tmp |= LC_REDO_EQ;
7511                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7512
7513                                 mdelay(100);
7514
7515                                 /* linkctl */
7516                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7517                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7518                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7519                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7520
7521                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7522                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7523                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7524                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7525
7526                                 /* linkctl2 */
7527                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7528                                 tmp16 &= ~((1 << 4) | (7 << 9));
7529                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7530                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7531
7532                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7533                                 tmp16 &= ~((1 << 4) | (7 << 9));
7534                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7535                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7536
7537                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7538                                 tmp &= ~LC_SET_QUIESCE;
7539                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7540                         }
7541                 }
7542         }
7543
7544         /* set the link speed */
7545         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7546         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7547         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7548
7549         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7550         tmp16 &= ~0xf;
7551         if (mask & DRM_PCIE_SPEED_80)
7552                 tmp16 |= 3; /* gen3 */
7553         else if (mask & DRM_PCIE_SPEED_50)
7554                 tmp16 |= 2; /* gen2 */
7555         else
7556                 tmp16 |= 1; /* gen1 */
7557         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7558
7559         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7560         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7561         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7562
7563         for (i = 0; i < rdev->usec_timeout; i++) {
7564                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7565                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7566                         break;
7567                 udelay(1);
7568         }
7569 }
7570
7571 static void si_program_aspm(struct radeon_device *rdev)
7572 {
7573         u32 data, orig;
7574         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7575         bool disable_clkreq = false;
7576
7577         if (radeon_aspm == 0)
7578                 return;
7579
7580         if (!(rdev->flags & RADEON_IS_PCIE))
7581                 return;
7582
7583         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7584         data &= ~LC_XMIT_N_FTS_MASK;
7585         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7586         if (orig != data)
7587                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7588
7589         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7590         data |= LC_GO_TO_RECOVERY;
7591         if (orig != data)
7592                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7593
7594         orig = data = RREG32_PCIE(PCIE_P_CNTL);
7595         data |= P_IGNORE_EDB_ERR;
7596         if (orig != data)
7597                 WREG32_PCIE(PCIE_P_CNTL, data);
7598
7599         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7600         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7601         data |= LC_PMI_TO_L1_DIS;
7602         if (!disable_l0s)
7603                 data |= LC_L0S_INACTIVITY(7);
7604
7605         if (!disable_l1) {
7606                 data |= LC_L1_INACTIVITY(7);
7607                 data &= ~LC_PMI_TO_L1_DIS;
7608                 if (orig != data)
7609                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7610
7611                 if (!disable_plloff_in_l1) {
7612                         bool clk_req_support;
7613
7614                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7615                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7616                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7617                         if (orig != data)
7618                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7619
7620                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7621                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7622                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7623                         if (orig != data)
7624                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7625
7626                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7627                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7628                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7629                         if (orig != data)
7630                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7631
7632                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7633                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7634                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7635                         if (orig != data)
7636                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7637
7638                         if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7639                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7640                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7641                                 if (orig != data)
7642                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7643
7644                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7645                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7646                                 if (orig != data)
7647                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7648
7649                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7650                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7651                                 if (orig != data)
7652                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7653
7654                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7655                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7656                                 if (orig != data)
7657                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7658
7659                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7660                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7661                                 if (orig != data)
7662                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7663
7664                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7665                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7666                                 if (orig != data)
7667                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7668
7669                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7670                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7671                                 if (orig != data)
7672                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7673
7674                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7675                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7676                                 if (orig != data)
7677                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7678                         }
7679                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7680                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7681                         data |= LC_DYN_LANES_PWR_STATE(3);
7682                         if (orig != data)
7683                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7684
7685                         orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7686                         data &= ~LS2_EXIT_TIME_MASK;
7687                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7688                                 data |= LS2_EXIT_TIME(5);
7689                         if (orig != data)
7690                                 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7691
7692                         orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7693                         data &= ~LS2_EXIT_TIME_MASK;
7694                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7695                                 data |= LS2_EXIT_TIME(5);
7696                         if (orig != data)
7697                                 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7698
7699                         if (!disable_clkreq &&
7700                             !pci_is_root_bus(rdev->pdev->bus)) {
7701                                 struct pci_dev *root = rdev->pdev->bus->self;
7702                                 u32 lnkcap;
7703
7704                                 clk_req_support = false;
7705                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7706                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7707                                         clk_req_support = true;
7708                         } else {
7709                                 clk_req_support = false;
7710                         }
7711
7712                         if (clk_req_support) {
7713                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7714                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7715                                 if (orig != data)
7716                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7717
7718                                 orig = data = RREG32(THM_CLK_CNTL);
7719                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7720                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7721                                 if (orig != data)
7722                                         WREG32(THM_CLK_CNTL, data);
7723
7724                                 orig = data = RREG32(MISC_CLK_CNTL);
7725                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7726                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7727                                 if (orig != data)
7728                                         WREG32(MISC_CLK_CNTL, data);
7729
7730                                 orig = data = RREG32(CG_CLKPIN_CNTL);
7731                                 data &= ~BCLK_AS_XCLK;
7732                                 if (orig != data)
7733                                         WREG32(CG_CLKPIN_CNTL, data);
7734
7735                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7736                                 data &= ~FORCE_BIF_REFCLK_EN;
7737                                 if (orig != data)
7738                                         WREG32(CG_CLKPIN_CNTL_2, data);
7739
7740                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7741                                 data &= ~MPLL_CLKOUT_SEL_MASK;
7742                                 data |= MPLL_CLKOUT_SEL(4);
7743                                 if (orig != data)
7744                                         WREG32(MPLL_BYPASSCLK_SEL, data);
7745
7746                                 orig = data = RREG32(SPLL_CNTL_MODE);
7747                                 data &= ~SPLL_REFCLK_SEL_MASK;
7748                                 if (orig != data)
7749                                         WREG32(SPLL_CNTL_MODE, data);
7750                         }
7751                 }
7752         } else {
7753                 if (orig != data)
7754                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7755         }
7756
7757         orig = data = RREG32_PCIE(PCIE_CNTL2);
7758         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7759         if (orig != data)
7760                 WREG32_PCIE(PCIE_CNTL2, data);
7761
7762         if (!disable_l0s) {
7763                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7764                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7765                         data = RREG32_PCIE(PCIE_LC_STATUS1);
7766                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7767                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7768                                 data &= ~LC_L0S_INACTIVITY_MASK;
7769                                 if (orig != data)
7770                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7771                         }
7772                 }
7773         }
7774 }
7775
7776 int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7777 {
7778         unsigned i;
7779
7780         /* make sure VCEPLL_CTLREQ is deasserted */
7781         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7782
7783         mdelay(10);
7784
7785         /* assert UPLL_CTLREQ */
7786         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7787
7788         /* wait for CTLACK and CTLACK2 to get asserted */
7789         for (i = 0; i < 100; ++i) {
7790                 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7791                 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7792                         break;
7793                 mdelay(10);
7794         }
7795
7796         /* deassert UPLL_CTLREQ */
7797         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7798
7799         if (i == 100) {
7800                 DRM_ERROR("Timeout setting UVD clocks!\n");
7801                 return -ETIMEDOUT;
7802         }
7803
7804         return 0;
7805 }
7806
7807 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7808 {
7809         unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7810         int r;
7811
7812         /* bypass evclk and ecclk with bclk */
7813         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7814                      EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7815                      ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7816
7817         /* put PLL in bypass mode */
7818         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7819                      ~VCEPLL_BYPASS_EN_MASK);
7820
7821         if (!evclk || !ecclk) {
7822                 /* keep the Bypass mode, put PLL to sleep */
7823                 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7824                              ~VCEPLL_SLEEP_MASK);
7825                 return 0;
7826         }
7827
7828         r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7829                                           16384, 0x03FFFFFF, 0, 128, 5,
7830                                           &fb_div, &evclk_div, &ecclk_div);
7831         if (r)
7832                 return r;
7833
7834         /* set RESET_ANTI_MUX to 0 */
7835         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7836
7837         /* set VCO_MODE to 1 */
7838         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7839                      ~VCEPLL_VCO_MODE_MASK);
7840
7841         /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7842         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7843                      ~VCEPLL_SLEEP_MASK);
7844         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7845
7846         /* deassert VCEPLL_RESET */
7847         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7848
7849         mdelay(1);
7850
7851         r = si_vce_send_vcepll_ctlreq(rdev);
7852         if (r)
7853                 return r;
7854
7855         /* assert VCEPLL_RESET again */
7856         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7857
7858         /* disable spread spectrum. */
7859         WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7860
7861         /* set feedback divider */
7862         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7863
7864         /* set ref divider to 0 */
7865         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7866
7867         /* set PDIV_A and PDIV_B */
7868         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7869                      VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7870                      ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7871
7872         /* give the PLL some time to settle */
7873         mdelay(15);
7874
7875         /* deassert PLL_RESET */
7876         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7877
7878         mdelay(15);
7879
7880         /* switch from bypass mode to normal mode */
7881         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7882
7883         r = si_vce_send_vcepll_ctlreq(rdev);
7884         if (r)
7885                 return r;
7886
7887         /* switch VCLK and DCLK selection */
7888         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7889                      EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7890                      ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7891
7892         mdelay(100);
7893
7894         return 0;
7895 }