kernel/drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include "radeon_audio.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 static const u32 crtc_offsets[6] =
  39 {
  40         EVERGREEN_CRTC0_REGISTER_OFFSET,
  41         EVERGREEN_CRTC1_REGISTER_OFFSET,
  42         EVERGREEN_CRTC2_REGISTER_OFFSET,
  43         EVERGREEN_CRTC3_REGISTER_OFFSET,
  44         EVERGREEN_CRTC4_REGISTER_OFFSET,
  45         EVERGREEN_CRTC5_REGISTER_OFFSET
  46 };
  47
  48 #include "clearstate_evergreen.h"
  49
  50 static const u32 sumo_rlc_save_restore_register_list[] =
  51 {
  52         0x98fc,
  53         0x9830,
  54         0x9834,
  55         0x9838,
  56         0x9870,
  57         0x9874,
  58         0x8a14,
  59         0x8b24,
  60         0x8bcc,
  61         0x8b10,
  62         0x8d00,
  63         0x8d04,
  64         0x8c00,
  65         0x8c04,
  66         0x8c08,
  67         0x8c0c,
  68         0x8d8c,
  69         0x8c20,
  70         0x8c24,
  71         0x8c28,
  72         0x8c18,
  73         0x8c1c,
  74         0x8cf0,
  75         0x8e2c,
  76         0x8e38,
  77         0x8c30,
  78         0x9508,
  79         0x9688,
  80         0x9608,
  81         0x960c,
  82         0x9610,
  83         0x9614,
  84         0x88c4,
  85         0x88d4,
  86         0xa008,
  87         0x900c,
  88         0x9100,
  89         0x913c,
  90         0x98f8,
  91         0x98f4,
  92         0x9b7c,
  93         0x3f8c,
  94         0x8950,
  95         0x8954,
  96         0x8a18,
  97         0x8b28,
  98         0x9144,
  99         0x9148,
 100         0x914c,
 101         0x3f90,
 102         0x3f94,
 103         0x915c,
 104         0x9160,
 105         0x9178,
 106         0x917c,
 107         0x9180,
 108         0x918c,
 109         0x9190,
 110         0x9194,
 111         0x9198,
 112         0x919c,
 113         0x91a8,
 114         0x91ac,
 115         0x91b0,
 116         0x91b4,
 117         0x91b8,
 118         0x91c4,
 119         0x91c8,
 120         0x91cc,
 121         0x91d0,
 122         0x91d4,
 123         0x91e0,
 124         0x91e4,
 125         0x91ec,
 126         0x91f0,
 127         0x91f4,
 128         0x9200,
 129         0x9204,
 130         0x929c,
 131         0x9150,
 132         0x802c,
 133 };
 134
 135 static void evergreen_gpu_init(struct radeon_device *rdev);
 136 void evergreen_fini(struct radeon_device *rdev);
 137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 138 void evergreen_program_aspm(struct radeon_device *rdev);
 139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 140                                      int ring, u32 cp_int_cntl);
 141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 142                                    u32 status, u32 addr);
 143 void cik_init_cp_pg_table(struct radeon_device *rdev);
 144
 145 extern u32 si_get_csb_size(struct radeon_device *rdev);
 146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 149 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 150
 151 static const u32 evergreen_golden_registers[] =
 152 {
 153         0x3f90, 0xffff0000, 0xff000000,
 154         0x9148, 0xffff0000, 0xff000000,
 155         0x3f94, 0xffff0000, 0xff000000,
 156         0x914c, 0xffff0000, 0xff000000,
 157         0x9b7c, 0xffffffff, 0x00000000,
 158         0x8a14, 0xffffffff, 0x00000007,
 159         0x8b10, 0xffffffff, 0x00000000,
 160         0x960c, 0xffffffff, 0x54763210,
 161         0x88c4, 0xffffffff, 0x000000c2,
 162         0x88d4, 0xffffffff, 0x00000010,
 163         0x8974, 0xffffffff, 0x00000000,
 164         0xc78, 0x00000080, 0x00000080,
 165         0x5eb4, 0xffffffff, 0x00000002,
 166         0x5e78, 0xffffffff, 0x001000f0,
 167         0x6104, 0x01000300, 0x00000000,
 168         0x5bc0, 0x00300000, 0x00000000,
 169         0x7030, 0xffffffff, 0x00000011,
 170         0x7c30, 0xffffffff, 0x00000011,
 171         0x10830, 0xffffffff, 0x00000011,
 172         0x11430, 0xffffffff, 0x00000011,
 173         0x12030, 0xffffffff, 0x00000011,
 174         0x12c30, 0xffffffff, 0x00000011,
 175         0xd02c, 0xffffffff, 0x08421000,
 176         0x240c, 0xffffffff, 0x00000380,
 177         0x8b24, 0xffffffff, 0x00ff0fff,
 178         0x28a4c, 0x06000000, 0x06000000,
 179         0x10c, 0x00000001, 0x00000001,
 180         0x8d00, 0xffffffff, 0x100e4848,
 181         0x8d04, 0xffffffff, 0x00164745,
 182         0x8c00, 0xffffffff, 0xe4000003,
 183         0x8c04, 0xffffffff, 0x40600060,
 184         0x8c08, 0xffffffff, 0x001c001c,
 185         0x8cf0, 0xffffffff, 0x08e00620,
 186         0x8c20, 0xffffffff, 0x00800080,
 187         0x8c24, 0xffffffff, 0x00800080,
 188         0x8c18, 0xffffffff, 0x20202078,
 189         0x8c1c, 0xffffffff, 0x00001010,
 190         0x28350, 0xffffffff, 0x00000000,
 191         0xa008, 0xffffffff, 0x00010000,
 192         0x5c4, 0xffffffff, 0x00000001,
 193         0x9508, 0xffffffff, 0x00000002,
 194         0x913c, 0x0000000f, 0x0000000a
 195 };
 196
 197 static const u32 evergreen_golden_registers2[] =
 198 {
 199         0x2f4c, 0xffffffff, 0x00000000,
 200         0x54f4, 0xffffffff, 0x00000000,
 201         0x54f0, 0xffffffff, 0x00000000,
 202         0x5498, 0xffffffff, 0x00000000,
 203         0x549c, 0xffffffff, 0x00000000,
 204         0x5494, 0xffffffff, 0x00000000,
 205         0x53cc, 0xffffffff, 0x00000000,
 206         0x53c8, 0xffffffff, 0x00000000,
 207         0x53c4, 0xffffffff, 0x00000000,
 208         0x53c0, 0xffffffff, 0x00000000,
 209         0x53bc, 0xffffffff, 0x00000000,
 210         0x53b8, 0xffffffff, 0x00000000,
 211         0x53b4, 0xffffffff, 0x00000000,
 212         0x53b0, 0xffffffff, 0x00000000
 213 };
 214
 215 static const u32 cypress_mgcg_init[] =
 216 {
 217         0x802c, 0xffffffff, 0xc0000000,
 218         0x5448, 0xffffffff, 0x00000100,
 219         0x55e4, 0xffffffff, 0x00000100,
 220         0x160c, 0xffffffff, 0x00000100,
 221         0x5644, 0xffffffff, 0x00000100,
 222         0xc164, 0xffffffff, 0x00000100,
 223         0x8a18, 0xffffffff, 0x00000100,
 224         0x897c, 0xffffffff, 0x06000100,
 225         0x8b28, 0xffffffff, 0x00000100,
 226         0x9144, 0xffffffff, 0x00000100,
 227         0x9a60, 0xffffffff, 0x00000100,
 228         0x9868, 0xffffffff, 0x00000100,
 229         0x8d58, 0xffffffff, 0x00000100,
 230         0x9510, 0xffffffff, 0x00000100,
 231         0x949c, 0xffffffff, 0x00000100,
 232         0x9654, 0xffffffff, 0x00000100,
 233         0x9030, 0xffffffff, 0x00000100,
 234         0x9034, 0xffffffff, 0x00000100,
 235         0x9038, 0xffffffff, 0x00000100,
 236         0x903c, 0xffffffff, 0x00000100,
 237         0x9040, 0xffffffff, 0x00000100,
 238         0xa200, 0xffffffff, 0x00000100,
 239         0xa204, 0xffffffff, 0x00000100,
 240         0xa208, 0xffffffff, 0x00000100,
 241         0xa20c, 0xffffffff, 0x00000100,
 242         0x971c, 0xffffffff, 0x00000100,
 243         0x977c, 0xffffffff, 0x00000100,
 244         0x3f80, 0xffffffff, 0x00000100,
 245         0xa210, 0xffffffff, 0x00000100,
 246         0xa214, 0xffffffff, 0x00000100,
 247         0x4d8, 0xffffffff, 0x00000100,
 248         0x9784, 0xffffffff, 0x00000100,
 249         0x9698, 0xffffffff, 0x00000100,
 250         0x4d4, 0xffffffff, 0x00000200,
 251         0x30cc, 0xffffffff, 0x00000100,
 252         0xd0c0, 0xffffffff, 0xff000100,
 253         0x802c, 0xffffffff, 0x40000000,
 254         0x915c, 0xffffffff, 0x00010000,
 255         0x9160, 0xffffffff, 0x00030002,
 256         0x9178, 0xffffffff, 0x00070000,
 257         0x917c, 0xffffffff, 0x00030002,
 258         0x9180, 0xffffffff, 0x00050004,
 259         0x918c, 0xffffffff, 0x00010006,
 260         0x9190, 0xffffffff, 0x00090008,
 261         0x9194, 0xffffffff, 0x00070000,
 262         0x9198, 0xffffffff, 0x00030002,
 263         0x919c, 0xffffffff, 0x00050004,
 264         0x91a8, 0xffffffff, 0x00010006,
 265         0x91ac, 0xffffffff, 0x00090008,
 266         0x91b0, 0xffffffff, 0x00070000,
 267         0x91b4, 0xffffffff, 0x00030002,
 268         0x91b8, 0xffffffff, 0x00050004,
 269         0x91c4, 0xffffffff, 0x00010006,
 270         0x91c8, 0xffffffff, 0x00090008,
 271         0x91cc, 0xffffffff, 0x00070000,
 272         0x91d0, 0xffffffff, 0x00030002,
 273         0x91d4, 0xffffffff, 0x00050004,
 274         0x91e0, 0xffffffff, 0x00010006,
 275         0x91e4, 0xffffffff, 0x00090008,
 276         0x91e8, 0xffffffff, 0x00000000,
 277         0x91ec, 0xffffffff, 0x00070000,
 278         0x91f0, 0xffffffff, 0x00030002,
 279         0x91f4, 0xffffffff, 0x00050004,
 280         0x9200, 0xffffffff, 0x00010006,
 281         0x9204, 0xffffffff, 0x00090008,
 282         0x9208, 0xffffffff, 0x00070000,
 283         0x920c, 0xffffffff, 0x00030002,
 284         0x9210, 0xffffffff, 0x00050004,
 285         0x921c, 0xffffffff, 0x00010006,
 286         0x9220, 0xffffffff, 0x00090008,
 287         0x9224, 0xffffffff, 0x00070000,
 288         0x9228, 0xffffffff, 0x00030002,
 289         0x922c, 0xffffffff, 0x00050004,
 290         0x9238, 0xffffffff, 0x00010006,
 291         0x923c, 0xffffffff, 0x00090008,
 292         0x9240, 0xffffffff, 0x00070000,
 293         0x9244, 0xffffffff, 0x00030002,
 294         0x9248, 0xffffffff, 0x00050004,
 295         0x9254, 0xffffffff, 0x00010006,
 296         0x9258, 0xffffffff, 0x00090008,
 297         0x925c, 0xffffffff, 0x00070000,
 298         0x9260, 0xffffffff, 0x00030002,
 299         0x9264, 0xffffffff, 0x00050004,
 300         0x9270, 0xffffffff, 0x00010006,
 301         0x9274, 0xffffffff, 0x00090008,
 302         0x9278, 0xffffffff, 0x00070000,
 303         0x927c, 0xffffffff, 0x00030002,
 304         0x9280, 0xffffffff, 0x00050004,
 305         0x928c, 0xffffffff, 0x00010006,
 306         0x9290, 0xffffffff, 0x00090008,
 307         0x9294, 0xffffffff, 0x00000000,
 308         0x929c, 0xffffffff, 0x00000001,
 309         0x802c, 0xffffffff, 0x40010000,
 310         0x915c, 0xffffffff, 0x00010000,
 311         0x9160, 0xffffffff, 0x00030002,
 312         0x9178, 0xffffffff, 0x00070000,
 313         0x917c, 0xffffffff, 0x00030002,
 314         0x9180, 0xffffffff, 0x00050004,
 315         0x918c, 0xffffffff, 0x00010006,
 316         0x9190, 0xffffffff, 0x00090008,
 317         0x9194, 0xffffffff, 0x00070000,
 318         0x9198, 0xffffffff, 0x00030002,
 319         0x919c, 0xffffffff, 0x00050004,
 320         0x91a8, 0xffffffff, 0x00010006,
 321         0x91ac, 0xffffffff, 0x00090008,
 322         0x91b0, 0xffffffff, 0x00070000,
 323         0x91b4, 0xffffffff, 0x00030002,
 324         0x91b8, 0xffffffff, 0x00050004,
 325         0x91c4, 0xffffffff, 0x00010006,
 326         0x91c8, 0xffffffff, 0x00090008,
 327         0x91cc, 0xffffffff, 0x00070000,
 328         0x91d0, 0xffffffff, 0x00030002,
 329         0x91d4, 0xffffffff, 0x00050004,
 330         0x91e0, 0xffffffff, 0x00010006,
 331         0x91e4, 0xffffffff, 0x00090008,
 332         0x91e8, 0xffffffff, 0x00000000,
 333         0x91ec, 0xffffffff, 0x00070000,
 334         0x91f0, 0xffffffff, 0x00030002,
 335         0x91f4, 0xffffffff, 0x00050004,
 336         0x9200, 0xffffffff, 0x00010006,
 337         0x9204, 0xffffffff, 0x00090008,
 338         0x9208, 0xffffffff, 0x00070000,
 339         0x920c, 0xffffffff, 0x00030002,
 340         0x9210, 0xffffffff, 0x00050004,
 341         0x921c, 0xffffffff, 0x00010006,
 342         0x9220, 0xffffffff, 0x00090008,
 343         0x9224, 0xffffffff, 0x00070000,
 344         0x9228, 0xffffffff, 0x00030002,
 345         0x922c, 0xffffffff, 0x00050004,
 346         0x9238, 0xffffffff, 0x00010006,
 347         0x923c, 0xffffffff, 0x00090008,
 348         0x9240, 0xffffffff, 0x00070000,
 349         0x9244, 0xffffffff, 0x00030002,
 350         0x9248, 0xffffffff, 0x00050004,
 351         0x9254, 0xffffffff, 0x00010006,
 352         0x9258, 0xffffffff, 0x00090008,
 353         0x925c, 0xffffffff, 0x00070000,
 354         0x9260, 0xffffffff, 0x00030002,
 355         0x9264, 0xffffffff, 0x00050004,
 356         0x9270, 0xffffffff, 0x00010006,
 357         0x9274, 0xffffffff, 0x00090008,
 358         0x9278, 0xffffffff, 0x00070000,
 359         0x927c, 0xffffffff, 0x00030002,
 360         0x9280, 0xffffffff, 0x00050004,
 361         0x928c, 0xffffffff, 0x00010006,
 362         0x9290, 0xffffffff, 0x00090008,
 363         0x9294, 0xffffffff, 0x00000000,
 364         0x929c, 0xffffffff, 0x00000001,
 365         0x802c, 0xffffffff, 0xc0000000
 366 };
 367
 368 static const u32 redwood_mgcg_init[] =
 369 {
 370         0x802c, 0xffffffff, 0xc0000000,
 371         0x5448, 0xffffffff, 0x00000100,
 372         0x55e4, 0xffffffff, 0x00000100,
 373         0x160c, 0xffffffff, 0x00000100,
 374         0x5644, 0xffffffff, 0x00000100,
 375         0xc164, 0xffffffff, 0x00000100,
 376         0x8a18, 0xffffffff, 0x00000100,
 377         0x897c, 0xffffffff, 0x06000100,
 378         0x8b28, 0xffffffff, 0x00000100,
 379         0x9144, 0xffffffff, 0x00000100,
 380         0x9a60, 0xffffffff, 0x00000100,
 381         0x9868, 0xffffffff, 0x00000100,
 382         0x8d58, 0xffffffff, 0x00000100,
 383         0x9510, 0xffffffff, 0x00000100,
 384         0x949c, 0xffffffff, 0x00000100,
 385         0x9654, 0xffffffff, 0x00000100,
 386         0x9030, 0xffffffff, 0x00000100,
 387         0x9034, 0xffffffff, 0x00000100,
 388         0x9038, 0xffffffff, 0x00000100,
 389         0x903c, 0xffffffff, 0x00000100,
 390         0x9040, 0xffffffff, 0x00000100,
 391         0xa200, 0xffffffff, 0x00000100,
 392         0xa204, 0xffffffff, 0x00000100,
 393         0xa208, 0xffffffff, 0x00000100,
 394         0xa20c, 0xffffffff, 0x00000100,
 395         0x971c, 0xffffffff, 0x00000100,
 396         0x977c, 0xffffffff, 0x00000100,
 397         0x3f80, 0xffffffff, 0x00000100,
 398         0xa210, 0xffffffff, 0x00000100,
 399         0xa214, 0xffffffff, 0x00000100,
 400         0x4d8, 0xffffffff, 0x00000100,
 401         0x9784, 0xffffffff, 0x00000100,
 402         0x9698, 0xffffffff, 0x00000100,
 403         0x4d4, 0xffffffff, 0x00000200,
 404         0x30cc, 0xffffffff, 0x00000100,
 405         0xd0c0, 0xffffffff, 0xff000100,
 406         0x802c, 0xffffffff, 0x40000000,
 407         0x915c, 0xffffffff, 0x00010000,
 408         0x9160, 0xffffffff, 0x00030002,
 409         0x9178, 0xffffffff, 0x00070000,
 410         0x917c, 0xffffffff, 0x00030002,
 411         0x9180, 0xffffffff, 0x00050004,
 412         0x918c, 0xffffffff, 0x00010006,
 413         0x9190, 0xffffffff, 0x00090008,
 414         0x9194, 0xffffffff, 0x00070000,
 415         0x9198, 0xffffffff, 0x00030002,
 416         0x919c, 0xffffffff, 0x00050004,
 417         0x91a8, 0xffffffff, 0x00010006,
 418         0x91ac, 0xffffffff, 0x00090008,
 419         0x91b0, 0xffffffff, 0x00070000,
 420         0x91b4, 0xffffffff, 0x00030002,
 421         0x91b8, 0xffffffff, 0x00050004,
 422         0x91c4, 0xffffffff, 0x00010006,
 423         0x91c8, 0xffffffff, 0x00090008,
 424         0x91cc, 0xffffffff, 0x00070000,
 425         0x91d0, 0xffffffff, 0x00030002,
 426         0x91d4, 0xffffffff, 0x00050004,
 427         0x91e0, 0xffffffff, 0x00010006,
 428         0x91e4, 0xffffffff, 0x00090008,
 429         0x91e8, 0xffffffff, 0x00000000,
 430         0x91ec, 0xffffffff, 0x00070000,
 431         0x91f0, 0xffffffff, 0x00030002,
 432         0x91f4, 0xffffffff, 0x00050004,
 433         0x9200, 0xffffffff, 0x00010006,
 434         0x9204, 0xffffffff, 0x00090008,
 435         0x9294, 0xffffffff, 0x00000000,
 436         0x929c, 0xffffffff, 0x00000001,
 437         0x802c, 0xffffffff, 0xc0000000
 438 };
 439
 440 static const u32 cedar_golden_registers[] =
 441 {
 442         0x3f90, 0xffff0000, 0xff000000,
 443         0x9148, 0xffff0000, 0xff000000,
 444         0x3f94, 0xffff0000, 0xff000000,
 445         0x914c, 0xffff0000, 0xff000000,
 446         0x9b7c, 0xffffffff, 0x00000000,
 447         0x8a14, 0xffffffff, 0x00000007,
 448         0x8b10, 0xffffffff, 0x00000000,
 449         0x960c, 0xffffffff, 0x54763210,
 450         0x88c4, 0xffffffff, 0x000000c2,
 451         0x88d4, 0xffffffff, 0x00000000,
 452         0x8974, 0xffffffff, 0x00000000,
 453         0xc78, 0x00000080, 0x00000080,
 454         0x5eb4, 0xffffffff, 0x00000002,
 455         0x5e78, 0xffffffff, 0x001000f0,
 456         0x6104, 0x01000300, 0x00000000,
 457         0x5bc0, 0x00300000, 0x00000000,
 458         0x7030, 0xffffffff, 0x00000011,
 459         0x7c30, 0xffffffff, 0x00000011,
 460         0x10830, 0xffffffff, 0x00000011,
 461         0x11430, 0xffffffff, 0x00000011,
 462         0xd02c, 0xffffffff, 0x08421000,
 463         0x240c, 0xffffffff, 0x00000380,
 464         0x8b24, 0xffffffff, 0x00ff0fff,
 465         0x28a4c, 0x06000000, 0x06000000,
 466         0x10c, 0x00000001, 0x00000001,
 467         0x8d00, 0xffffffff, 0x100e4848,
 468         0x8d04, 0xffffffff, 0x00164745,
 469         0x8c00, 0xffffffff, 0xe4000003,
 470         0x8c04, 0xffffffff, 0x40600060,
 471         0x8c08, 0xffffffff, 0x001c001c,
 472         0x8cf0, 0xffffffff, 0x08e00410,
 473         0x8c20, 0xffffffff, 0x00800080,
 474         0x8c24, 0xffffffff, 0x00800080,
 475         0x8c18, 0xffffffff, 0x20202078,
 476         0x8c1c, 0xffffffff, 0x00001010,
 477         0x28350, 0xffffffff, 0x00000000,
 478         0xa008, 0xffffffff, 0x00010000,
 479         0x5c4, 0xffffffff, 0x00000001,
 480         0x9508, 0xffffffff, 0x00000002
 481 };
 482
 483 static const u32 cedar_mgcg_init[] =
 484 {
 485         0x802c, 0xffffffff, 0xc0000000,
 486         0x5448, 0xffffffff, 0x00000100,
 487         0x55e4, 0xffffffff, 0x00000100,
 488         0x160c, 0xffffffff, 0x00000100,
 489         0x5644, 0xffffffff, 0x00000100,
 490         0xc164, 0xffffffff, 0x00000100,
 491         0x8a18, 0xffffffff, 0x00000100,
 492         0x897c, 0xffffffff, 0x06000100,
 493         0x8b28, 0xffffffff, 0x00000100,
 494         0x9144, 0xffffffff, 0x00000100,
 495         0x9a60, 0xffffffff, 0x00000100,
 496         0x9868, 0xffffffff, 0x00000100,
 497         0x8d58, 0xffffffff, 0x00000100,
 498         0x9510, 0xffffffff, 0x00000100,
 499         0x949c, 0xffffffff, 0x00000100,
 500         0x9654, 0xffffffff, 0x00000100,
 501         0x9030, 0xffffffff, 0x00000100,
 502         0x9034, 0xffffffff, 0x00000100,
 503         0x9038, 0xffffffff, 0x00000100,
 504         0x903c, 0xffffffff, 0x00000100,
 505         0x9040, 0xffffffff, 0x00000100,
 506         0xa200, 0xffffffff, 0x00000100,
 507         0xa204, 0xffffffff, 0x00000100,
 508         0xa208, 0xffffffff, 0x00000100,
 509         0xa20c, 0xffffffff, 0x00000100,
 510         0x971c, 0xffffffff, 0x00000100,
 511         0x977c, 0xffffffff, 0x00000100,
 512         0x3f80, 0xffffffff, 0x00000100,
 513         0xa210, 0xffffffff, 0x00000100,
 514         0xa214, 0xffffffff, 0x00000100,
 515         0x4d8, 0xffffffff, 0x00000100,
 516         0x9784, 0xffffffff, 0x00000100,
 517         0x9698, 0xffffffff, 0x00000100,
 518         0x4d4, 0xffffffff, 0x00000200,
 519         0x30cc, 0xffffffff, 0x00000100,
 520         0xd0c0, 0xffffffff, 0xff000100,
 521         0x802c, 0xffffffff, 0x40000000,
 522         0x915c, 0xffffffff, 0x00010000,
 523         0x9178, 0xffffffff, 0x00050000,
 524         0x917c, 0xffffffff, 0x00030002,
 525         0x918c, 0xffffffff, 0x00010004,
 526         0x9190, 0xffffffff, 0x00070006,
 527         0x9194, 0xffffffff, 0x00050000,
 528         0x9198, 0xffffffff, 0x00030002,
 529         0x91a8, 0xffffffff, 0x00010004,
 530         0x91ac, 0xffffffff, 0x00070006,
 531         0x91e8, 0xffffffff, 0x00000000,
 532         0x9294, 0xffffffff, 0x00000000,
 533         0x929c, 0xffffffff, 0x00000001,
 534         0x802c, 0xffffffff, 0xc0000000
 535 };
 536
 537 static const u32 juniper_mgcg_init[] =
 538 {
 539         0x802c, 0xffffffff, 0xc0000000,
 540         0x5448, 0xffffffff, 0x00000100,
 541         0x55e4, 0xffffffff, 0x00000100,
 542         0x160c, 0xffffffff, 0x00000100,
 543         0x5644, 0xffffffff, 0x00000100,
 544         0xc164, 0xffffffff, 0x00000100,
 545         0x8a18, 0xffffffff, 0x00000100,
 546         0x897c, 0xffffffff, 0x06000100,
 547         0x8b28, 0xffffffff, 0x00000100,
 548         0x9144, 0xffffffff, 0x00000100,
 549         0x9a60, 0xffffffff, 0x00000100,
 550         0x9868, 0xffffffff, 0x00000100,
 551         0x8d58, 0xffffffff, 0x00000100,
 552         0x9510, 0xffffffff, 0x00000100,
 553         0x949c, 0xffffffff, 0x00000100,
 554         0x9654, 0xffffffff, 0x00000100,
 555         0x9030, 0xffffffff, 0x00000100,
 556         0x9034, 0xffffffff, 0x00000100,
 557         0x9038, 0xffffffff, 0x00000100,
 558         0x903c, 0xffffffff, 0x00000100,
 559         0x9040, 0xffffffff, 0x00000100,
 560         0xa200, 0xffffffff, 0x00000100,
 561         0xa204, 0xffffffff, 0x00000100,
 562         0xa208, 0xffffffff, 0x00000100,
 563         0xa20c, 0xffffffff, 0x00000100,
 564         0x971c, 0xffffffff, 0x00000100,
 565         0xd0c0, 0xffffffff, 0xff000100,
 566         0x802c, 0xffffffff, 0x40000000,
 567         0x915c, 0xffffffff, 0x00010000,
 568         0x9160, 0xffffffff, 0x00030002,
 569         0x9178, 0xffffffff, 0x00070000,
 570         0x917c, 0xffffffff, 0x00030002,
 571         0x9180, 0xffffffff, 0x00050004,
 572         0x918c, 0xffffffff, 0x00010006,
 573         0x9190, 0xffffffff, 0x00090008,
 574         0x9194, 0xffffffff, 0x00070000,
 575         0x9198, 0xffffffff, 0x00030002,
 576         0x919c, 0xffffffff, 0x00050004,
 577         0x91a8, 0xffffffff, 0x00010006,
 578         0x91ac, 0xffffffff, 0x00090008,
 579         0x91b0, 0xffffffff, 0x00070000,
 580         0x91b4, 0xffffffff, 0x00030002,
 581         0x91b8, 0xffffffff, 0x00050004,
 582         0x91c4, 0xffffffff, 0x00010006,
 583         0x91c8, 0xffffffff, 0x00090008,
 584         0x91cc, 0xffffffff, 0x00070000,
 585         0x91d0, 0xffffffff, 0x00030002,
 586         0x91d4, 0xffffffff, 0x00050004,
 587         0x91e0, 0xffffffff, 0x00010006,
 588         0x91e4, 0xffffffff, 0x00090008,
 589         0x91e8, 0xffffffff, 0x00000000,
 590         0x91ec, 0xffffffff, 0x00070000,
 591         0x91f0, 0xffffffff, 0x00030002,
 592         0x91f4, 0xffffffff, 0x00050004,
 593         0x9200, 0xffffffff, 0x00010006,
 594         0x9204, 0xffffffff, 0x00090008,
 595         0x9208, 0xffffffff, 0x00070000,
 596         0x920c, 0xffffffff, 0x00030002,
 597         0x9210, 0xffffffff, 0x00050004,
 598         0x921c, 0xffffffff, 0x00010006,
 599         0x9220, 0xffffffff, 0x00090008,
 600         0x9224, 0xffffffff, 0x00070000,
 601         0x9228, 0xffffffff, 0x00030002,
 602         0x922c, 0xffffffff, 0x00050004,
 603         0x9238, 0xffffffff, 0x00010006,
 604         0x923c, 0xffffffff, 0x00090008,
 605         0x9240, 0xffffffff, 0x00070000,
 606         0x9244, 0xffffffff, 0x00030002,
 607         0x9248, 0xffffffff, 0x00050004,
 608         0x9254, 0xffffffff, 0x00010006,
 609         0x9258, 0xffffffff, 0x00090008,
 610         0x925c, 0xffffffff, 0x00070000,
 611         0x9260, 0xffffffff, 0x00030002,
 612         0x9264, 0xffffffff, 0x00050004,
 613         0x9270, 0xffffffff, 0x00010006,
 614         0x9274, 0xffffffff, 0x00090008,
 615         0x9278, 0xffffffff, 0x00070000,
 616         0x927c, 0xffffffff, 0x00030002,
 617         0x9280, 0xffffffff, 0x00050004,
 618         0x928c, 0xffffffff, 0x00010006,
 619         0x9290, 0xffffffff, 0x00090008,
 620         0x9294, 0xffffffff, 0x00000000,
 621         0x929c, 0xffffffff, 0x00000001,
 622         0x802c, 0xffffffff, 0xc0000000,
 623         0x977c, 0xffffffff, 0x00000100,
 624         0x3f80, 0xffffffff, 0x00000100,
 625         0xa210, 0xffffffff, 0x00000100,
 626         0xa214, 0xffffffff, 0x00000100,
 627         0x4d8, 0xffffffff, 0x00000100,
 628         0x9784, 0xffffffff, 0x00000100,
 629         0x9698, 0xffffffff, 0x00000100,
 630         0x4d4, 0xffffffff, 0x00000200,
 631         0x30cc, 0xffffffff, 0x00000100,
 632         0x802c, 0xffffffff, 0xc0000000
 633 };
 634
 635 static const u32 supersumo_golden_registers[] =
 636 {
 637         0x5eb4, 0xffffffff, 0x00000002,
 638         0x5c4, 0xffffffff, 0x00000001,
 639         0x7030, 0xffffffff, 0x00000011,
 640         0x7c30, 0xffffffff, 0x00000011,
 641         0x6104, 0x01000300, 0x00000000,
 642         0x5bc0, 0x00300000, 0x00000000,
 643         0x8c04, 0xffffffff, 0x40600060,
 644         0x8c08, 0xffffffff, 0x001c001c,
 645         0x8c20, 0xffffffff, 0x00800080,
 646         0x8c24, 0xffffffff, 0x00800080,
 647         0x8c18, 0xffffffff, 0x20202078,
 648         0x8c1c, 0xffffffff, 0x00001010,
 649         0x918c, 0xffffffff, 0x00010006,
 650         0x91a8, 0xffffffff, 0x00010006,
 651         0x91c4, 0xffffffff, 0x00010006,
 652         0x91e0, 0xffffffff, 0x00010006,
 653         0x9200, 0xffffffff, 0x00010006,
 654         0x9150, 0xffffffff, 0x6e944040,
 655         0x917c, 0xffffffff, 0x00030002,
 656         0x9180, 0xffffffff, 0x00050004,
 657         0x9198, 0xffffffff, 0x00030002,
 658         0x919c, 0xffffffff, 0x00050004,
 659         0x91b4, 0xffffffff, 0x00030002,
 660         0x91b8, 0xffffffff, 0x00050004,
 661         0x91d0, 0xffffffff, 0x00030002,
 662         0x91d4, 0xffffffff, 0x00050004,
 663         0x91f0, 0xffffffff, 0x00030002,
 664         0x91f4, 0xffffffff, 0x00050004,
 665         0x915c, 0xffffffff, 0x00010000,
 666         0x9160, 0xffffffff, 0x00030002,
 667         0x3f90, 0xffff0000, 0xff000000,
 668         0x9178, 0xffffffff, 0x00070000,
 669         0x9194, 0xffffffff, 0x00070000,
 670         0x91b0, 0xffffffff, 0x00070000,
 671         0x91cc, 0xffffffff, 0x00070000,
 672         0x91ec, 0xffffffff, 0x00070000,
 673         0x9148, 0xffff0000, 0xff000000,
 674         0x9190, 0xffffffff, 0x00090008,
 675         0x91ac, 0xffffffff, 0x00090008,
 676         0x91c8, 0xffffffff, 0x00090008,
 677         0x91e4, 0xffffffff, 0x00090008,
 678         0x9204, 0xffffffff, 0x00090008,
 679         0x3f94, 0xffff0000, 0xff000000,
 680         0x914c, 0xffff0000, 0xff000000,
 681         0x929c, 0xffffffff, 0x00000001,
 682         0x8a18, 0xffffffff, 0x00000100,
 683         0x8b28, 0xffffffff, 0x00000100,
 684         0x9144, 0xffffffff, 0x00000100,
 685         0x5644, 0xffffffff, 0x00000100,
 686         0x9b7c, 0xffffffff, 0x00000000,
 687         0x8030, 0xffffffff, 0x0000100a,
 688         0x8a14, 0xffffffff, 0x00000007,
 689         0x8b24, 0xffffffff, 0x00ff0fff,
 690         0x8b10, 0xffffffff, 0x00000000,
 691         0x28a4c, 0x06000000, 0x06000000,
 692         0x4d8, 0xffffffff, 0x00000100,
 693         0x913c, 0xffff000f, 0x0100000a,
 694         0x960c, 0xffffffff, 0x54763210,
 695         0x88c4, 0xffffffff, 0x000000c2,
 696         0x88d4, 0xffffffff, 0x00000010,
 697         0x8974, 0xffffffff, 0x00000000,
 698         0xc78, 0x00000080, 0x00000080,
 699         0x5e78, 0xffffffff, 0x001000f0,
 700         0xd02c, 0xffffffff, 0x08421000,
 701         0xa008, 0xffffffff, 0x00010000,
 702         0x8d00, 0xffffffff, 0x100e4848,
 703         0x8d04, 0xffffffff, 0x00164745,
 704         0x8c00, 0xffffffff, 0xe4000003,
 705         0x8cf0, 0x1fffffff, 0x08e00620,
 706         0x28350, 0xffffffff, 0x00000000,
 707         0x9508, 0xffffffff, 0x00000002
 708 };
 709
 710 static const u32 sumo_golden_registers[] =
 711 {
 712         0x900c, 0x00ffffff, 0x0017071f,
 713         0x8c18, 0xffffffff, 0x10101060,
 714         0x8c1c, 0xffffffff, 0x00001010,
 715         0x8c30, 0x0000000f, 0x00000005,
 716         0x9688, 0x0000000f, 0x00000007
 717 };
 718
 719 static const u32 wrestler_golden_registers[] =
 720 {
 721         0x5eb4, 0xffffffff, 0x00000002,
 722         0x5c4, 0xffffffff, 0x00000001,
 723         0x7030, 0xffffffff, 0x00000011,
 724         0x7c30, 0xffffffff, 0x00000011,
 725         0x6104, 0x01000300, 0x00000000,
 726         0x5bc0, 0x00300000, 0x00000000,
 727         0x918c, 0xffffffff, 0x00010006,
 728         0x91a8, 0xffffffff, 0x00010006,
 729         0x9150, 0xffffffff, 0x6e944040,
 730         0x917c, 0xffffffff, 0x00030002,
 731         0x9198, 0xffffffff, 0x00030002,
 732         0x915c, 0xffffffff, 0x00010000,
 733         0x3f90, 0xffff0000, 0xff000000,
 734         0x9178, 0xffffffff, 0x00070000,
 735         0x9194, 0xffffffff, 0x00070000,
 736         0x9148, 0xffff0000, 0xff000000,
 737         0x9190, 0xffffffff, 0x00090008,
 738         0x91ac, 0xffffffff, 0x00090008,
 739         0x3f94, 0xffff0000, 0xff000000,
 740         0x914c, 0xffff0000, 0xff000000,
 741         0x929c, 0xffffffff, 0x00000001,
 742         0x8a18, 0xffffffff, 0x00000100,
 743         0x8b28, 0xffffffff, 0x00000100,
 744         0x9144, 0xffffffff, 0x00000100,
 745         0x9b7c, 0xffffffff, 0x00000000,
 746         0x8030, 0xffffffff, 0x0000100a,
 747         0x8a14, 0xffffffff, 0x00000001,
 748         0x8b24, 0xffffffff, 0x00ff0fff,
 749         0x8b10, 0xffffffff, 0x00000000,
 750         0x28a4c, 0x06000000, 0x06000000,
 751         0x4d8, 0xffffffff, 0x00000100,
 752         0x913c, 0xffff000f, 0x0100000a,
 753         0x960c, 0xffffffff, 0x54763210,
 754         0x88c4, 0xffffffff, 0x000000c2,
 755         0x88d4, 0xffffffff, 0x00000010,
 756         0x8974, 0xffffffff, 0x00000000,
 757         0xc78, 0x00000080, 0x00000080,
 758         0x5e78, 0xffffffff, 0x001000f0,
 759         0xd02c, 0xffffffff, 0x08421000,
 760         0xa008, 0xffffffff, 0x00010000,
 761         0x8d00, 0xffffffff, 0x100e4848,
 762         0x8d04, 0xffffffff, 0x00164745,
 763         0x8c00, 0xffffffff, 0xe4000003,
 764         0x8cf0, 0x1fffffff, 0x08e00410,
 765         0x28350, 0xffffffff, 0x00000000,
 766         0x9508, 0xffffffff, 0x00000002,
 767         0x900c, 0xffffffff, 0x0017071f,
 768         0x8c18, 0xffffffff, 0x10101060,
 769         0x8c1c, 0xffffffff, 0x00001010
 770 };
 771
 772 static const u32 barts_golden_registers[] =
 773 {
 774         0x5eb4, 0xffffffff, 0x00000002,
 775         0x5e78, 0x8f311ff1, 0x001000f0,
 776         0x3f90, 0xffff0000, 0xff000000,
 777         0x9148, 0xffff0000, 0xff000000,
 778         0x3f94, 0xffff0000, 0xff000000,
 779         0x914c, 0xffff0000, 0xff000000,
 780         0xc78, 0x00000080, 0x00000080,
 781         0xbd4, 0x70073777, 0x00010001,
 782         0xd02c, 0xbfffff1f, 0x08421000,
 783         0xd0b8, 0x03773777, 0x02011003,
 784         0x5bc0, 0x00200000, 0x50100000,
 785         0x98f8, 0x33773777, 0x02011003,
 786         0x98fc, 0xffffffff, 0x76543210,
 787         0x7030, 0x31000311, 0x00000011,
 788         0x2f48, 0x00000007, 0x02011003,
 789         0x6b28, 0x00000010, 0x00000012,
 790         0x7728, 0x00000010, 0x00000012,
 791         0x10328, 0x00000010, 0x00000012,
 792         0x10f28, 0x00000010, 0x00000012,
 793         0x11b28, 0x00000010, 0x00000012,
 794         0x12728, 0x00000010, 0x00000012,
 795         0x240c, 0x000007ff, 0x00000380,
 796         0x8a14, 0xf000001f, 0x00000007,
 797         0x8b24, 0x3fff3fff, 0x00ff0fff,
 798         0x8b10, 0x0000ff0f, 0x00000000,
 799         0x28a4c, 0x07ffffff, 0x06000000,
 800         0x10c, 0x00000001, 0x00010003,
 801         0xa02c, 0xffffffff, 0x0000009b,
 802         0x913c, 0x0000000f, 0x0100000a,
 803         0x8d00, 0xffff7f7f, 0x100e4848,
 804         0x8d04, 0x00ffffff, 0x00164745,
 805         0x8c00, 0xfffc0003, 0xe4000003,
 806         0x8c04, 0xf8ff00ff, 0x40600060,
 807         0x8c08, 0x00ff00ff, 0x001c001c,
 808         0x8cf0, 0x1fff1fff, 0x08e00620,
 809         0x8c20, 0x0fff0fff, 0x00800080,
 810         0x8c24, 0x0fff0fff, 0x00800080,
 811         0x8c18, 0xffffffff, 0x20202078,
 812         0x8c1c, 0x0000ffff, 0x00001010,
 813         0x28350, 0x00000f01, 0x00000000,
 814         0x9508, 0x3700001f, 0x00000002,
 815         0x960c, 0xffffffff, 0x54763210,
 816         0x88c4, 0x001f3ae3, 0x000000c2,
 817         0x88d4, 0x0000001f, 0x00000010,
 818         0x8974, 0xffffffff, 0x00000000
 819 };
 820
 821 static const u32 turks_golden_registers[] =
 822 {
 823         0x5eb4, 0xffffffff, 0x00000002,
 824         0x5e78, 0x8f311ff1, 0x001000f0,
 825         0x8c8, 0x00003000, 0x00001070,
 826         0x8cc, 0x000fffff, 0x00040035,
 827         0x3f90, 0xffff0000, 0xfff00000,
 828         0x9148, 0xffff0000, 0xfff00000,
 829         0x3f94, 0xffff0000, 0xfff00000,
 830         0x914c, 0xffff0000, 0xfff00000,
 831         0xc78, 0x00000080, 0x00000080,
 832         0xbd4, 0x00073007, 0x00010002,
 833         0xd02c, 0xbfffff1f, 0x08421000,
 834         0xd0b8, 0x03773777, 0x02010002,
 835         0x5bc0, 0x00200000, 0x50100000,
 836         0x98f8, 0x33773777, 0x00010002,
 837         0x98fc, 0xffffffff, 0x33221100,
 838         0x7030, 0x31000311, 0x00000011,
 839         0x2f48, 0x33773777, 0x00010002,
 840         0x6b28, 0x00000010, 0x00000012,
 841         0x7728, 0x00000010, 0x00000012,
 842         0x10328, 0x00000010, 0x00000012,
 843         0x10f28, 0x00000010, 0x00000012,
 844         0x11b28, 0x00000010, 0x00000012,
 845         0x12728, 0x00000010, 0x00000012,
 846         0x240c, 0x000007ff, 0x00000380,
 847         0x8a14, 0xf000001f, 0x00000007,
 848         0x8b24, 0x3fff3fff, 0x00ff0fff,
 849         0x8b10, 0x0000ff0f, 0x00000000,
 850         0x28a4c, 0x07ffffff, 0x06000000,
 851         0x10c, 0x00000001, 0x00010003,
 852         0xa02c, 0xffffffff, 0x0000009b,
 853         0x913c, 0x0000000f, 0x0100000a,
 854         0x8d00, 0xffff7f7f, 0x100e4848,
 855         0x8d04, 0x00ffffff, 0x00164745,
 856         0x8c00, 0xfffc0003, 0xe4000003,
 857         0x8c04, 0xf8ff00ff, 0x40600060,
 858         0x8c08, 0x00ff00ff, 0x001c001c,
 859         0x8cf0, 0x1fff1fff, 0x08e00410,
 860         0x8c20, 0x0fff0fff, 0x00800080,
 861         0x8c24, 0x0fff0fff, 0x00800080,
 862         0x8c18, 0xffffffff, 0x20202078,
 863         0x8c1c, 0x0000ffff, 0x00001010,
 864         0x28350, 0x00000f01, 0x00000000,
 865         0x9508, 0x3700001f, 0x00000002,
 866         0x960c, 0xffffffff, 0x54763210,
 867         0x88c4, 0x001f3ae3, 0x000000c2,
 868         0x88d4, 0x0000001f, 0x00000010,
 869         0x8974, 0xffffffff, 0x00000000
 870 };
 871
 872 static const u32 caicos_golden_registers[] =
 873 {
 874         0x5eb4, 0xffffffff, 0x00000002,
 875         0x5e78, 0x8f311ff1, 0x001000f0,
 876         0x8c8, 0x00003420, 0x00001450,
 877         0x8cc, 0x000fffff, 0x00040035,
 878         0x3f90, 0xffff0000, 0xfffc0000,
 879         0x9148, 0xffff0000, 0xfffc0000,
 880         0x3f94, 0xffff0000, 0xfffc0000,
 881         0x914c, 0xffff0000, 0xfffc0000,
 882         0xc78, 0x00000080, 0x00000080,
 883         0xbd4, 0x00073007, 0x00010001,
 884         0xd02c, 0xbfffff1f, 0x08421000,
 885         0xd0b8, 0x03773777, 0x02010001,
 886         0x5bc0, 0x00200000, 0x50100000,
 887         0x98f8, 0x33773777, 0x02010001,
 888         0x98fc, 0xffffffff, 0x33221100,
 889         0x7030, 0x31000311, 0x00000011,
 890         0x2f48, 0x33773777, 0x02010001,
 891         0x6b28, 0x00000010, 0x00000012,
 892         0x7728, 0x00000010, 0x00000012,
 893         0x10328, 0x00000010, 0x00000012,
 894         0x10f28, 0x00000010, 0x00000012,
 895         0x11b28, 0x00000010, 0x00000012,
 896         0x12728, 0x00000010, 0x00000012,
 897         0x240c, 0x000007ff, 0x00000380,
 898         0x8a14, 0xf000001f, 0x00000001,
 899         0x8b24, 0x3fff3fff, 0x00ff0fff,
 900         0x8b10, 0x0000ff0f, 0x00000000,
 901         0x28a4c, 0x07ffffff, 0x06000000,
 902         0x10c, 0x00000001, 0x00010003,
 903         0xa02c, 0xffffffff, 0x0000009b,
 904         0x913c, 0x0000000f, 0x0100000a,
 905         0x8d00, 0xffff7f7f, 0x100e4848,
 906         0x8d04, 0x00ffffff, 0x00164745,
 907         0x8c00, 0xfffc0003, 0xe4000003,
 908         0x8c04, 0xf8ff00ff, 0x40600060,
 909         0x8c08, 0x00ff00ff, 0x001c001c,
 910         0x8cf0, 0x1fff1fff, 0x08e00410,
 911         0x8c20, 0x0fff0fff, 0x00800080,
 912         0x8c24, 0x0fff0fff, 0x00800080,
 913         0x8c18, 0xffffffff, 0x20202078,
 914         0x8c1c, 0x0000ffff, 0x00001010,
 915         0x28350, 0x00000f01, 0x00000000,
 916         0x9508, 0x3700001f, 0x00000002,
 917         0x960c, 0xffffffff, 0x54763210,
 918         0x88c4, 0x001f3ae3, 0x000000c2,
 919         0x88d4, 0x0000001f, 0x00000010,
 920         0x8974, 0xffffffff, 0x00000000
 921 };
 922
 923 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 924 {
 925         switch (rdev->family) {
 926         case CHIP_CYPRESS:
 927         case CHIP_HEMLOCK:
 928                 radeon_program_register_sequence(rdev,
 929                                                  evergreen_golden_registers,
 930                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 931                 radeon_program_register_sequence(rdev,
 932                                                  evergreen_golden_registers2,
 933                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 934                 radeon_program_register_sequence(rdev,
 935                                                  cypress_mgcg_init,
 936                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 937                 break;
 938         case CHIP_JUNIPER:
 939                 radeon_program_register_sequence(rdev,
 940                                                  evergreen_golden_registers,
 941                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 942                 radeon_program_register_sequence(rdev,
 943                                                  evergreen_golden_registers2,
 944                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 945                 radeon_program_register_sequence(rdev,
 946                                                  juniper_mgcg_init,
 947                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
 948                 break;
 949         case CHIP_REDWOOD:
 950                 radeon_program_register_sequence(rdev,
 951                                                  evergreen_golden_registers,
 952                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 953                 radeon_program_register_sequence(rdev,
 954                                                  evergreen_golden_registers2,
 955                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 956                 radeon_program_register_sequence(rdev,
 957                                                  redwood_mgcg_init,
 958                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
 959                 break;
 960         case CHIP_CEDAR:
 961                 radeon_program_register_sequence(rdev,
 962                                                  cedar_golden_registers,
 963                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
 964                 radeon_program_register_sequence(rdev,
 965                                                  evergreen_golden_registers2,
 966                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 967                 radeon_program_register_sequence(rdev,
 968                                                  cedar_mgcg_init,
 969                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
 970                 break;
 971         case CHIP_PALM:
 972                 radeon_program_register_sequence(rdev,
 973                                                  wrestler_golden_registers,
 974                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
 975                 break;
 976         case CHIP_SUMO:
 977                 radeon_program_register_sequence(rdev,
 978                                                  supersumo_golden_registers,
 979                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 980                 break;
 981         case CHIP_SUMO2:
 982                 radeon_program_register_sequence(rdev,
 983                                                  supersumo_golden_registers,
 984                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 985                 radeon_program_register_sequence(rdev,
 986                                                  sumo_golden_registers,
 987                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
 988                 break;
 989         case CHIP_BARTS:
 990                 radeon_program_register_sequence(rdev,
 991                                                  barts_golden_registers,
 992                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
 993                 break;
 994         case CHIP_TURKS:
 995                 radeon_program_register_sequence(rdev,
 996                                                  turks_golden_registers,
 997                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
 998                 break;
 999         case CHIP_CAICOS:
1000                 radeon_program_register_sequence(rdev,
1001                                                  caicos_golden_registers,
1002                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1003                 break;
1004         default:
1005                 break;
1006         }
1007 }
1008
1009 /**
1010  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1011  *
1012  * @rdev: radeon_device pointer
1013  * @reg: register offset in bytes
1014  * @val: register value
1015  *
1016  * Returns 0 for success or -EINVAL for an invalid register
1017  *
1018  */
1019 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1020                                         u32 reg, u32 *val)
1021 {
1022         switch (reg) {
1023         case GRBM_STATUS:
1024         case GRBM_STATUS_SE0:
1025         case GRBM_STATUS_SE1:
1026         case SRBM_STATUS:
1027         case SRBM_STATUS2:
1028         case DMA_STATUS_REG:
1029         case UVD_STATUS:
1030                 *val = RREG32(reg);
1031                 return 0;
1032         default:
1033                 return -EINVAL;
1034         }
1035 }
1036
1037 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1038                              unsigned *bankh, unsigned *mtaspect,
1039                              unsigned *tile_split)
1040 {
1041         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1042         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1043         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1044         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1045         switch (*bankw) {
1046         default:
1047         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1048         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1049         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1050         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1051         }
1052         switch (*bankh) {
1053         default:
1054         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1055         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1056         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1057         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1058         }
1059         switch (*mtaspect) {
1060         default:
1061         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1062         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1063         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1064         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1065         }
1066 }
1067
1068 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1069                               u32 cntl_reg, u32 status_reg)
1070 {
1071         int r, i;
1072         struct atom_clock_dividers dividers;
1073
1074         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1075                                            clock, false, &dividers);
1076         if (r)
1077                 return r;
1078
1079         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1080
1081         for (i = 0; i < 100; i++) {
1082                 if (RREG32(status_reg) & DCLK_STATUS)
1083                         break;
1084                 mdelay(10);
1085         }
1086         if (i == 100)
1087                 return -ETIMEDOUT;
1088
1089         return 0;
1090 }
1091
1092 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1093 {
1094         int r = 0;
1095         u32 cg_scratch = RREG32(CG_SCRATCH1);
1096
1097         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1098         if (r)
1099                 goto done;
1100         cg_scratch &= 0xffff0000;
1101         cg_scratch |= vclk / 100; /* Mhz */
1102
1103         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1104         if (r)
1105                 goto done;
1106         cg_scratch &= 0x0000ffff;
1107         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1108
1109 done:
1110         WREG32(CG_SCRATCH1, cg_scratch);
1111
1112         return r;
1113 }
1114
1115 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1116 {
1117         /* start off with something large */
1118         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1119         int r;
1120
1121         /* bypass vclk and dclk with bclk */
1122         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1123                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1124                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1125
1126         /* put PLL in bypass mode */
1127         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1128
1129         if (!vclk || !dclk) {
1130                 /* keep the Bypass mode, put PLL to sleep */
1131                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1132                 return 0;
1133         }
1134
1135         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1136                                           16384, 0x03FFFFFF, 0, 128, 5,
1137                                           &fb_div, &vclk_div, &dclk_div);
1138         if (r)
1139                 return r;
1140
1141         /* set VCO_MODE to 1 */
1142         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1143
1144         /* toggle UPLL_SLEEP to 1 then back to 0 */
1145         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1146         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1147
1148         /* deassert UPLL_RESET */
1149         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1150
1151         mdelay(1);
1152
1153         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1154         if (r)
1155                 return r;
1156
1157         /* assert UPLL_RESET again */
1158         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1159
1160         /* disable spread spectrum. */
1161         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1162
1163         /* set feedback divider */
1164         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1165
1166         /* set ref divider to 0 */
1167         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1168
1169         if (fb_div < 307200)
1170                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1171         else
1172                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1173
1174         /* set PDIV_A and PDIV_B */
1175         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1176                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1177                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1178
1179         /* give the PLL some time to settle */
1180         mdelay(15);
1181
1182         /* deassert PLL_RESET */
1183         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1184
1185         mdelay(15);
1186
1187         /* switch from bypass mode to normal mode */
1188         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1189
1190         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1191         if (r)
1192                 return r;
1193
1194         /* switch VCLK and DCLK selection */
1195         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1196                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1197                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1198
1199         mdelay(100);
1200
1201         return 0;
1202 }
1203
1204 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1205 {
1206         int readrq;
1207         u16 v;
1208
1209         readrq = pcie_get_readrq(rdev->pdev);
1210         v = ffs(readrq) - 8;
1211         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1212          * to avoid hangs or perfomance issues
1213          */
1214         if ((v == 0) || (v == 6) || (v == 7))
1215                 pcie_set_readrq(rdev->pdev, 512);
1216 }
1217
1218 void dce4_program_fmt(struct drm_encoder *encoder)
1219 {
1220         struct drm_device *dev = encoder->dev;
1221         struct radeon_device *rdev = dev->dev_private;
1222         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1223         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1224         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1225         int bpc = 0;
1226         u32 tmp = 0;
1227         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1228
1229         if (connector) {
1230                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1231                 bpc = radeon_get_monitor_bpc(connector);
1232                 dither = radeon_connector->dither;
1233         }
1234
1235         /* LVDS/eDP FMT is set up by atom */
1236         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1237                 return;
1238
1239         /* not needed for analog */
1240         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1241             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1242                 return;
1243
1244         if (bpc == 0)
1245                 return;
1246
1247         switch (bpc) {
1248         case 6:
1249                 if (dither == RADEON_FMT_DITHER_ENABLE)
1250                         /* XXX sort out optimal dither settings */
1251                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1252                                 FMT_SPATIAL_DITHER_EN);
1253                 else
1254                         tmp |= FMT_TRUNCATE_EN;
1255                 break;
1256         case 8:
1257                 if (dither == RADEON_FMT_DITHER_ENABLE)
1258                         /* XXX sort out optimal dither settings */
1259                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1260                                 FMT_RGB_RANDOM_ENABLE |
1261                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1262                 else
1263                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1264                 break;
1265         case 10:
1266         default:
1267                 /* not needed */
1268                 break;
1269         }
1270
1271         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1272 }
1273
1274 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1275 {
1276         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1277                 return true;
1278         else
1279                 return false;
1280 }
1281
1282 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1283 {
1284         u32 pos1, pos2;
1285
1286         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1287         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1288
1289         if (pos1 != pos2)
1290                 return true;
1291         else
1292                 return false;
1293 }
1294
1295 /**
1296  * dce4_wait_for_vblank - vblank wait asic callback.
1297  *
1298  * @rdev: radeon_device pointer
1299  * @crtc: crtc to wait for vblank on
1300  *
1301  * Wait for vblank on the requested crtc (evergreen+).
1302  */
1303 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1304 {
1305         unsigned i = 0;
1306
1307         if (crtc >= rdev->num_crtc)
1308                 return;
1309
1310         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1311                 return;
1312
1313         /* depending on when we hit vblank, we may be close to active; if so,
1314          * wait for another frame.
1315          */
1316         while (dce4_is_in_vblank(rdev, crtc)) {
1317                 if (i++ % 100 == 0) {
1318                         if (!dce4_is_counter_moving(rdev, crtc))
1319                                 break;
1320                 }
1321         }
1322
1323         while (!dce4_is_in_vblank(rdev, crtc)) {
1324                 if (i++ % 100 == 0) {
1325                         if (!dce4_is_counter_moving(rdev, crtc))
1326                                 break;
1327                 }
1328         }
1329 }
1330
1331 /**
1332  * evergreen_page_flip - pageflip callback.
1333  *
1334  * @rdev: radeon_device pointer
1335  * @crtc_id: crtc to cleanup pageflip on
1336  * @crtc_base: new address of the crtc (GPU MC address)
1337  *
1338  * Does the actual pageflip (evergreen+).
1339  * During vblank we take the crtc lock and wait for the update_pending
1340  * bit to go high, when it does, we release the lock, and allow the
1341  * double buffered update to take place.
1342  * Returns the current update pending status.
1343  */
1344 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1345 {
1346         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1347         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1348         int i;
1349
1350         /* Lock the graphics update lock */
1351         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1352         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1353
1354         /* update the scanout addresses */
1355         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1356                upper_32_bits(crtc_base));
1357         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1358                (u32)crtc_base);
1359
1360         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1361                upper_32_bits(crtc_base));
1362         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1363                (u32)crtc_base);
1364
1365         /* Wait for update_pending to go high. */
1366         for (i = 0; i < rdev->usec_timeout; i++) {
1367                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1368                         break;
1369                 udelay(1);
1370         }
1371         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1372
1373         /* Unlock the lock, so double-buffering can take place inside vblank */
1374         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1375         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1376 }
1377
1378 /**
1379  * evergreen_page_flip_pending - check if page flip is still pending
1380  *
1381  * @rdev: radeon_device pointer
1382  * @crtc_id: crtc to check
1383  *
1384  * Returns the current update pending status.
1385  */
1386 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1387 {
1388         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1389
1390         /* Return current update_pending status: */
1391         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1392                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1393 }
1394
1395 /* get temperature in millidegrees */
1396 int evergreen_get_temp(struct radeon_device *rdev)
1397 {
1398         u32 temp, toffset;
1399         int actual_temp = 0;
1400
1401         if (rdev->family == CHIP_JUNIPER) {
1402                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1403                         TOFFSET_SHIFT;
1404                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1405                         TS0_ADC_DOUT_SHIFT;
1406
1407                 if (toffset & 0x100)
1408                         actual_temp = temp / 2 - (0x200 - toffset);
1409                 else
1410                         actual_temp = temp / 2 + toffset;
1411
1412                 actual_temp = actual_temp * 1000;
1413
1414         } else {
1415                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1416                         ASIC_T_SHIFT;
1417
1418                 if (temp & 0x400)
1419                         actual_temp = -256;
1420                 else if (temp & 0x200)
1421                         actual_temp = 255;
1422                 else if (temp & 0x100) {
1423                         actual_temp = temp & 0x1ff;
1424                         actual_temp |= ~0x1ff;
1425                 } else
1426                         actual_temp = temp & 0xff;
1427
1428                 actual_temp = (actual_temp * 1000) / 2;
1429         }
1430
1431         return actual_temp;
1432 }
1433
1434 int sumo_get_temp(struct radeon_device *rdev)
1435 {
1436         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1437         int actual_temp = temp - 49;
1438
1439         return actual_temp * 1000;
1440 }
1441
1442 /**
1443  * sumo_pm_init_profile - Initialize power profiles callback.
1444  *
1445  * @rdev: radeon_device pointer
1446  *
1447  * Initialize the power states used in profile mode
1448  * (sumo, trinity, SI).
1449  * Used for profile mode only.
1450  */
1451 void sumo_pm_init_profile(struct radeon_device *rdev)
1452 {
1453         int idx;
1454
1455         /* default */
1456         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1457         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1458         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1459         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1460
1461         /* low,mid sh/mh */
1462         if (rdev->flags & RADEON_IS_MOBILITY)
1463                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1464         else
1465                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1466
1467         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1468         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1469         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1470         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1471
1472         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1473         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1474         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1475         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1476
1477         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1478         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1479         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1480         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1481
1482         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1486
1487         /* high sh/mh */
1488         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1489         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1490         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1491         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1492         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1493                 rdev->pm.power_state[idx].num_clock_modes - 1;
1494
1495         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1496         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1497         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1498         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1499                 rdev->pm.power_state[idx].num_clock_modes - 1;
1500 }
1501
1502 /**
1503  * btc_pm_init_profile - Initialize power profiles callback.
1504  *
1505  * @rdev: radeon_device pointer
1506  *
1507  * Initialize the power states used in profile mode
1508  * (BTC, cayman).
1509  * Used for profile mode only.
1510  */
1511 void btc_pm_init_profile(struct radeon_device *rdev)
1512 {
1513         int idx;
1514
1515         /* default */
1516         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1517         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1518         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1519         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1520         /* starting with BTC, there is one state that is used for both
1521          * MH and SH.  Difference is that we always use the high clock index for
1522          * mclk.
1523          */
1524         if (rdev->flags & RADEON_IS_MOBILITY)
1525                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1526         else
1527                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1528         /* low sh */
1529         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1530         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1531         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1532         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1533         /* mid sh */
1534         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1538         /* high sh */
1539         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1540         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1541         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1542         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1543         /* low mh */
1544         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1545         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1546         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1547         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1548         /* mid mh */
1549         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1550         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1551         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1552         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1553         /* high mh */
1554         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1558 }
1559
1560 /**
1561  * evergreen_pm_misc - set additional pm hw parameters callback.
1562  *
1563  * @rdev: radeon_device pointer
1564  *
1565  * Set non-clock parameters associated with a power state
1566  * (voltage, etc.) (evergreen+).
1567  */
1568 void evergreen_pm_misc(struct radeon_device *rdev)
1569 {
1570         int req_ps_idx = rdev->pm.requested_power_state_index;
1571         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1572         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1573         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1574
1575         if (voltage->type == VOLTAGE_SW) {
1576                 /* 0xff0x are flags rather then an actual voltage */
1577                 if ((voltage->voltage & 0xff00) == 0xff00)
1578                         return;
1579                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1580                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1581                         rdev->pm.current_vddc = voltage->voltage;
1582                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1583                 }
1584
1585                 /* starting with BTC, there is one state that is used for both
1586                  * MH and SH.  Difference is that we always use the high clock index for
1587                  * mclk and vddci.
1588                  */
1589                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1590                     (rdev->family >= CHIP_BARTS) &&
1591                     rdev->pm.active_crtc_count &&
1592                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1593                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1594                         voltage = &rdev->pm.power_state[req_ps_idx].
1595                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1596
1597                 /* 0xff0x are flags rather then an actual voltage */
1598                 if ((voltage->vddci & 0xff00) == 0xff00)
1599                         return;
1600                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1601                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1602                         rdev->pm.current_vddci = voltage->vddci;
1603                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1604                 }
1605         }
1606 }
1607
1608 /**
1609  * evergreen_pm_prepare - pre-power state change callback.
1610  *
1611  * @rdev: radeon_device pointer
1612  *
1613  * Prepare for a power state change (evergreen+).
1614  */
1615 void evergreen_pm_prepare(struct radeon_device *rdev)
1616 {
1617         struct drm_device *ddev = rdev->ddev;
1618         struct drm_crtc *crtc;
1619         struct radeon_crtc *radeon_crtc;
1620         u32 tmp;
1621
1622         /* disable any active CRTCs */
1623         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1624                 radeon_crtc = to_radeon_crtc(crtc);
1625                 if (radeon_crtc->enabled) {
1626                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1627                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1628                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1629                 }
1630         }
1631 }
1632
1633 /**
1634  * evergreen_pm_finish - post-power state change callback.
1635  *
1636  * @rdev: radeon_device pointer
1637  *
1638  * Clean up after a power state change (evergreen+).
1639  */
1640 void evergreen_pm_finish(struct radeon_device *rdev)
1641 {
1642         struct drm_device *ddev = rdev->ddev;
1643         struct drm_crtc *crtc;
1644         struct radeon_crtc *radeon_crtc;
1645         u32 tmp;
1646
1647         /* enable any active CRTCs */
1648         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1649                 radeon_crtc = to_radeon_crtc(crtc);
1650                 if (radeon_crtc->enabled) {
1651                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1652                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1653                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1654                 }
1655         }
1656 }
1657
1658 /**
1659  * evergreen_hpd_sense - hpd sense callback.
1660  *
1661  * @rdev: radeon_device pointer
1662  * @hpd: hpd (hotplug detect) pin
1663  *
1664  * Checks if a digital monitor is connected (evergreen+).
1665  * Returns true if connected, false if not connected.
1666  */
1667 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1668 {
1669         bool connected = false;
1670
1671         switch (hpd) {
1672         case RADEON_HPD_1:
1673                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1674                         connected = true;
1675                 break;
1676         case RADEON_HPD_2:
1677                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1678                         connected = true;
1679                 break;
1680         case RADEON_HPD_3:
1681                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1682                         connected = true;
1683                 break;
1684         case RADEON_HPD_4:
1685                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1686                         connected = true;
1687                 break;
1688         case RADEON_HPD_5:
1689                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1690                         connected = true;
1691                 break;
1692         case RADEON_HPD_6:
1693                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1694                         connected = true;
1695                 break;
1696         default:
1697                 break;
1698         }
1699
1700         return connected;
1701 }
1702
1703 /**
1704  * evergreen_hpd_set_polarity - hpd set polarity callback.
1705  *
1706  * @rdev: radeon_device pointer
1707  * @hpd: hpd (hotplug detect) pin
1708  *
1709  * Set the polarity of the hpd pin (evergreen+).
1710  */
1711 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1712                                 enum radeon_hpd_id hpd)
1713 {
1714         u32 tmp;
1715         bool connected = evergreen_hpd_sense(rdev, hpd);
1716
1717         switch (hpd) {
1718         case RADEON_HPD_1:
1719                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1720                 if (connected)
1721                         tmp &= ~DC_HPDx_INT_POLARITY;
1722                 else
1723                         tmp |= DC_HPDx_INT_POLARITY;
1724                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1725                 break;
1726         case RADEON_HPD_2:
1727                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1728                 if (connected)
1729                         tmp &= ~DC_HPDx_INT_POLARITY;
1730                 else
1731                         tmp |= DC_HPDx_INT_POLARITY;
1732                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1733                 break;
1734         case RADEON_HPD_3:
1735                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1736                 if (connected)
1737                         tmp &= ~DC_HPDx_INT_POLARITY;
1738                 else
1739                         tmp |= DC_HPDx_INT_POLARITY;
1740                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1741                 break;
1742         case RADEON_HPD_4:
1743                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1744                 if (connected)
1745                         tmp &= ~DC_HPDx_INT_POLARITY;
1746                 else
1747                         tmp |= DC_HPDx_INT_POLARITY;
1748                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1749                 break;
1750         case RADEON_HPD_5:
1751                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1752                 if (connected)
1753                         tmp &= ~DC_HPDx_INT_POLARITY;
1754                 else
1755                         tmp |= DC_HPDx_INT_POLARITY;
1756                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1757                         break;
1758         case RADEON_HPD_6:
1759                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1760                 if (connected)
1761                         tmp &= ~DC_HPDx_INT_POLARITY;
1762                 else
1763                         tmp |= DC_HPDx_INT_POLARITY;
1764                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1765                 break;
1766         default:
1767                 break;
1768         }
1769 }
1770
1771 /**
1772  * evergreen_hpd_init - hpd setup callback.
1773  *
1774  * @rdev: radeon_device pointer
1775  *
1776  * Setup the hpd pins used by the card (evergreen+).
1777  * Enable the pin, set the polarity, and enable the hpd interrupts.
1778  */
1779 void evergreen_hpd_init(struct radeon_device *rdev)
1780 {
1781         struct drm_device *dev = rdev->ddev;
1782         struct drm_connector *connector;
1783         unsigned enabled = 0;
1784         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1785                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1786
1787         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1788                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1789
1790                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1791                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1792                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1793                          * aux dp channel on imac and help (but not completely fix)
1794                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1795                          * also avoid interrupt storms during dpms.
1796                          */
1797                         continue;
1798                 }
1799                 switch (radeon_connector->hpd.hpd) {
1800                 case RADEON_HPD_1:
1801                         WREG32(DC_HPD1_CONTROL, tmp);
1802                         break;
1803                 case RADEON_HPD_2:
1804                         WREG32(DC_HPD2_CONTROL, tmp);
1805                         break;
1806                 case RADEON_HPD_3:
1807                         WREG32(DC_HPD3_CONTROL, tmp);
1808                         break;
1809                 case RADEON_HPD_4:
1810                         WREG32(DC_HPD4_CONTROL, tmp);
1811                         break;
1812                 case RADEON_HPD_5:
1813                         WREG32(DC_HPD5_CONTROL, tmp);
1814                         break;
1815                 case RADEON_HPD_6:
1816                         WREG32(DC_HPD6_CONTROL, tmp);
1817                         break;
1818                 default:
1819                         break;
1820                 }
1821                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1822                 enabled |= 1 << radeon_connector->hpd.hpd;
1823         }
1824         radeon_irq_kms_enable_hpd(rdev, enabled);
1825 }
1826
1827 /**
1828  * evergreen_hpd_fini - hpd tear down callback.
1829  *
1830  * @rdev: radeon_device pointer
1831  *
1832  * Tear down the hpd pins used by the card (evergreen+).
1833  * Disable the hpd interrupts.
1834  */
1835 void evergreen_hpd_fini(struct radeon_device *rdev)
1836 {
1837         struct drm_device *dev = rdev->ddev;
1838         struct drm_connector *connector;
1839         unsigned disabled = 0;
1840
1841         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1842                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1843                 switch (radeon_connector->hpd.hpd) {
1844                 case RADEON_HPD_1:
1845                         WREG32(DC_HPD1_CONTROL, 0);
1846                         break;
1847                 case RADEON_HPD_2:
1848                         WREG32(DC_HPD2_CONTROL, 0);
1849                         break;
1850                 case RADEON_HPD_3:
1851                         WREG32(DC_HPD3_CONTROL, 0);
1852                         break;
1853                 case RADEON_HPD_4:
1854                         WREG32(DC_HPD4_CONTROL, 0);
1855                         break;
1856                 case RADEON_HPD_5:
1857                         WREG32(DC_HPD5_CONTROL, 0);
1858                         break;
1859                 case RADEON_HPD_6:
1860                         WREG32(DC_HPD6_CONTROL, 0);
1861                         break;
1862                 default:
1863                         break;
1864                 }
1865                 disabled |= 1 << radeon_connector->hpd.hpd;
1866         }
1867         radeon_irq_kms_disable_hpd(rdev, disabled);
1868 }
1869
1870 /* watermark setup */
1871
1872 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1873                                         struct radeon_crtc *radeon_crtc,
1874                                         struct drm_display_mode *mode,
1875                                         struct drm_display_mode *other_mode)
1876 {
1877         u32 tmp, buffer_alloc, i;
1878         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1879         /*
1880          * Line Buffer Setup
1881          * There are 3 line buffers, each one shared by 2 display controllers.
1882          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1883          * the display controllers.  The paritioning is done via one of four
1884          * preset allocations specified in bits 2:0:
1885          * first display controller
1886          *  0 - first half of lb (3840 * 2)
1887          *  1 - first 3/4 of lb (5760 * 2)
1888          *  2 - whole lb (7680 * 2), other crtc must be disabled
1889          *  3 - first 1/4 of lb (1920 * 2)
1890          * second display controller
1891          *  4 - second half of lb (3840 * 2)
1892          *  5 - second 3/4 of lb (5760 * 2)
1893          *  6 - whole lb (7680 * 2), other crtc must be disabled
1894          *  7 - last 1/4 of lb (1920 * 2)
1895          */
1896         /* this can get tricky if we have two large displays on a paired group
1897          * of crtcs.  Ideally for multiple large displays we'd assign them to
1898          * non-linked crtcs for maximum line buffer allocation.
1899          */
1900         if (radeon_crtc->base.enabled && mode) {
1901                 if (other_mode) {
1902                         tmp = 0; /* 1/2 */
1903                         buffer_alloc = 1;
1904                 } else {
1905                         tmp = 2; /* whole */
1906                         buffer_alloc = 2;
1907                 }
1908         } else {
1909                 tmp = 0;
1910                 buffer_alloc = 0;
1911         }
1912
1913         /* second controller of the pair uses second half of the lb */
1914         if (radeon_crtc->crtc_id % 2)
1915                 tmp += 4;
1916         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1917
1918         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1919                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1920                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1921                 for (i = 0; i < rdev->usec_timeout; i++) {
1922                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1923                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1924                                 break;
1925                         udelay(1);
1926                 }
1927         }
1928
1929         if (radeon_crtc->base.enabled && mode) {
1930                 switch (tmp) {
1931                 case 0:
1932                 case 4:
1933                 default:
1934                         if (ASIC_IS_DCE5(rdev))
1935                                 return 4096 * 2;
1936                         else
1937                                 return 3840 * 2;
1938                 case 1:
1939                 case 5:
1940                         if (ASIC_IS_DCE5(rdev))
1941                                 return 6144 * 2;
1942                         else
1943                                 return 5760 * 2;
1944                 case 2:
1945                 case 6:
1946                         if (ASIC_IS_DCE5(rdev))
1947                                 return 8192 * 2;
1948                         else
1949                                 return 7680 * 2;
1950                 case 3:
1951                 case 7:
1952                         if (ASIC_IS_DCE5(rdev))
1953                                 return 2048 * 2;
1954                         else
1955                                 return 1920 * 2;
1956                 }
1957         }
1958
1959         /* controller not enabled, so no lb used */
1960         return 0;
1961 }
1962
1963 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1964 {
1965         u32 tmp = RREG32(MC_SHARED_CHMAP);
1966
1967         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1968         case 0:
1969         default:
1970                 return 1;
1971         case 1:
1972                 return 2;
1973         case 2:
1974                 return 4;
1975         case 3:
1976                 return 8;
1977         }
1978 }
1979
1980 struct evergreen_wm_params {
1981         u32 dram_channels; /* number of dram channels */
1982         u32 yclk;          /* bandwidth per dram data pin in kHz */
1983         u32 sclk;          /* engine clock in kHz */
1984         u32 disp_clk;      /* display clock in kHz */
1985         u32 src_width;     /* viewport width */
1986         u32 active_time;   /* active display time in ns */
1987         u32 blank_time;    /* blank time in ns */
1988         bool interlaced;    /* mode is interlaced */
1989         fixed20_12 vsc;    /* vertical scale ratio */
1990         u32 num_heads;     /* number of active crtcs */
1991         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1992         u32 lb_size;       /* line buffer allocated to pipe */
1993         u32 vtaps;         /* vertical scaler taps */
1994 };
1995
1996 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1997 {
1998         /* Calculate DRAM Bandwidth and the part allocated to display. */
1999         fixed20_12 dram_efficiency; /* 0.7 */
2000         fixed20_12 yclk, dram_channels, bandwidth;
2001         fixed20_12 a;
2002
2003         a.full = dfixed_const(1000);
2004         yclk.full = dfixed_const(wm->yclk);
2005         yclk.full = dfixed_div(yclk, a);
2006         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2007         a.full = dfixed_const(10);
2008         dram_efficiency.full = dfixed_const(7);
2009         dram_efficiency.full = dfixed_div(dram_efficiency, a);
2010         bandwidth.full = dfixed_mul(dram_channels, yclk);
2011         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2012
2013         return dfixed_trunc(bandwidth);
2014 }
2015
2016 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2017 {
2018         /* Calculate DRAM Bandwidth and the part allocated to display. */
2019         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2020         fixed20_12 yclk, dram_channels, bandwidth;
2021         fixed20_12 a;
2022
2023         a.full = dfixed_const(1000);
2024         yclk.full = dfixed_const(wm->yclk);
2025         yclk.full = dfixed_div(yclk, a);
2026         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2027         a.full = dfixed_const(10);
2028         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2029         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2030         bandwidth.full = dfixed_mul(dram_channels, yclk);
2031         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2032
2033         return dfixed_trunc(bandwidth);
2034 }
2035
2036 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2037 {
2038         /* Calculate the display Data return Bandwidth */
2039         fixed20_12 return_efficiency; /* 0.8 */
2040         fixed20_12 sclk, bandwidth;
2041         fixed20_12 a;
2042
2043         a.full = dfixed_const(1000);
2044         sclk.full = dfixed_const(wm->sclk);
2045         sclk.full = dfixed_div(sclk, a);
2046         a.full = dfixed_const(10);
2047         return_efficiency.full = dfixed_const(8);
2048         return_efficiency.full = dfixed_div(return_efficiency, a);
2049         a.full = dfixed_const(32);
2050         bandwidth.full = dfixed_mul(a, sclk);
2051         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2052
2053         return dfixed_trunc(bandwidth);
2054 }
2055
2056 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2057 {
2058         /* Calculate the DMIF Request Bandwidth */
2059         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2060         fixed20_12 disp_clk, bandwidth;
2061         fixed20_12 a;
2062
2063         a.full = dfixed_const(1000);
2064         disp_clk.full = dfixed_const(wm->disp_clk);
2065         disp_clk.full = dfixed_div(disp_clk, a);
2066         a.full = dfixed_const(10);
2067         disp_clk_request_efficiency.full = dfixed_const(8);
2068         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2069         a.full = dfixed_const(32);
2070         bandwidth.full = dfixed_mul(a, disp_clk);
2071         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2072
2073         return dfixed_trunc(bandwidth);
2074 }
2075
2076 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2077 {
2078         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2079         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2080         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2081         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2082
2083         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2084 }
2085
2086 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2087 {
2088         /* Calculate the display mode Average Bandwidth
2089          * DisplayMode should contain the source and destination dimensions,
2090          * timing, etc.
2091          */
2092         fixed20_12 bpp;
2093         fixed20_12 line_time;
2094         fixed20_12 src_width;
2095         fixed20_12 bandwidth;
2096         fixed20_12 a;
2097
2098         a.full = dfixed_const(1000);
2099         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2100         line_time.full = dfixed_div(line_time, a);
2101         bpp.full = dfixed_const(wm->bytes_per_pixel);
2102         src_width.full = dfixed_const(wm->src_width);
2103         bandwidth.full = dfixed_mul(src_width, bpp);
2104         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2105         bandwidth.full = dfixed_div(bandwidth, line_time);
2106
2107         return dfixed_trunc(bandwidth);
2108 }
2109
2110 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2111 {
2112         /* First calcualte the latency in ns */
2113         u32 mc_latency = 2000; /* 2000 ns. */
2114         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2115         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2116         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2117         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2118         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2119                 (wm->num_heads * cursor_line_pair_return_time);
2120         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2121         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2122         fixed20_12 a, b, c;
2123
2124         if (wm->num_heads == 0)
2125                 return 0;
2126
2127         a.full = dfixed_const(2);
2128         b.full = dfixed_const(1);
2129         if ((wm->vsc.full > a.full) ||
2130             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2131             (wm->vtaps >= 5) ||
2132             ((wm->vsc.full >= a.full) && wm->interlaced))
2133                 max_src_lines_per_dst_line = 4;
2134         else
2135                 max_src_lines_per_dst_line = 2;
2136
2137         a.full = dfixed_const(available_bandwidth);
2138         b.full = dfixed_const(wm->num_heads);
2139         a.full = dfixed_div(a, b);
2140
2141         b.full = dfixed_const(1000);
2142         c.full = dfixed_const(wm->disp_clk);
2143         b.full = dfixed_div(c, b);
2144         c.full = dfixed_const(wm->bytes_per_pixel);
2145         b.full = dfixed_mul(b, c);
2146
2147         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2148
2149         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2150         b.full = dfixed_const(1000);
2151         c.full = dfixed_const(lb_fill_bw);
2152         b.full = dfixed_div(c, b);
2153         a.full = dfixed_div(a, b);
2154         line_fill_time = dfixed_trunc(a);
2155
2156         if (line_fill_time < wm->active_time)
2157                 return latency;
2158         else
2159                 return latency + (line_fill_time - wm->active_time);
2160
2161 }
2162
2163 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2164 {
2165         if (evergreen_average_bandwidth(wm) <=
2166             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2167                 return true;
2168         else
2169                 return false;
2170 };
2171
2172 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2173 {
2174         if (evergreen_average_bandwidth(wm) <=
2175             (evergreen_available_bandwidth(wm) / wm->num_heads))
2176                 return true;
2177         else
2178                 return false;
2179 };
2180
2181 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2182 {
2183         u32 lb_partitions = wm->lb_size / wm->src_width;
2184         u32 line_time = wm->active_time + wm->blank_time;
2185         u32 latency_tolerant_lines;
2186         u32 latency_hiding;
2187         fixed20_12 a;
2188
2189         a.full = dfixed_const(1);
2190         if (wm->vsc.full > a.full)
2191                 latency_tolerant_lines = 1;
2192         else {
2193                 if (lb_partitions <= (wm->vtaps + 1))
2194                         latency_tolerant_lines = 1;
2195                 else
2196                         latency_tolerant_lines = 2;
2197         }
2198
2199         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2200
2201         if (evergreen_latency_watermark(wm) <= latency_hiding)
2202                 return true;
2203         else
2204                 return false;
2205 }
2206
2207 static void evergreen_program_watermarks(struct radeon_device *rdev,
2208                                          struct radeon_crtc *radeon_crtc,
2209                                          u32 lb_size, u32 num_heads)
2210 {
2211         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2212         struct evergreen_wm_params wm_low, wm_high;
2213         u32 dram_channels;
2214         u32 pixel_period;
2215         u32 line_time = 0;
2216         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2217         u32 priority_a_mark = 0, priority_b_mark = 0;
2218         u32 priority_a_cnt = PRIORITY_OFF;
2219         u32 priority_b_cnt = PRIORITY_OFF;
2220         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2221         u32 tmp, arb_control3;
2222         fixed20_12 a, b, c;
2223
2224         if (radeon_crtc->base.enabled && num_heads && mode) {
2225                 pixel_period = 1000000 / (u32)mode->clock;
2226                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2227                 priority_a_cnt = 0;
2228                 priority_b_cnt = 0;
2229                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2230
2231                 /* watermark for high clocks */
2232                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2233                         wm_high.yclk =
2234                                 radeon_dpm_get_mclk(rdev, false) * 10;
2235                         wm_high.sclk =
2236                                 radeon_dpm_get_sclk(rdev, false) * 10;
2237                 } else {
2238                         wm_high.yclk = rdev->pm.current_mclk * 10;
2239                         wm_high.sclk = rdev->pm.current_sclk * 10;
2240                 }
2241
2242                 wm_high.disp_clk = mode->clock;
2243                 wm_high.src_width = mode->crtc_hdisplay;
2244                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2245                 wm_high.blank_time = line_time - wm_high.active_time;
2246                 wm_high.interlaced = false;
2247                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2248                         wm_high.interlaced = true;
2249                 wm_high.vsc = radeon_crtc->vsc;
2250                 wm_high.vtaps = 1;
2251                 if (radeon_crtc->rmx_type != RMX_OFF)
2252                         wm_high.vtaps = 2;
2253                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2254                 wm_high.lb_size = lb_size;
2255                 wm_high.dram_channels = dram_channels;
2256                 wm_high.num_heads = num_heads;
2257
2258                 /* watermark for low clocks */
2259                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2260                         wm_low.yclk =
2261                                 radeon_dpm_get_mclk(rdev, true) * 10;
2262                         wm_low.sclk =
2263                                 radeon_dpm_get_sclk(rdev, true) * 10;
2264                 } else {
2265                         wm_low.yclk = rdev->pm.current_mclk * 10;
2266                         wm_low.sclk = rdev->pm.current_sclk * 10;
2267                 }
2268
2269                 wm_low.disp_clk = mode->clock;
2270                 wm_low.src_width = mode->crtc_hdisplay;
2271                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2272                 wm_low.blank_time = line_time - wm_low.active_time;
2273                 wm_low.interlaced = false;
2274                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2275                         wm_low.interlaced = true;
2276                 wm_low.vsc = radeon_crtc->vsc;
2277                 wm_low.vtaps = 1;
2278                 if (radeon_crtc->rmx_type != RMX_OFF)
2279                         wm_low.vtaps = 2;
2280                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2281                 wm_low.lb_size = lb_size;
2282                 wm_low.dram_channels = dram_channels;
2283                 wm_low.num_heads = num_heads;
2284
2285                 /* set for high clocks */
2286                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2287                 /* set for low clocks */
2288                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2289
2290                 /* possibly force display priority to high */
2291                 /* should really do this at mode validation time... */
2292                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2293                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2294                     !evergreen_check_latency_hiding(&wm_high) ||
2295                     (rdev->disp_priority == 2)) {
2296                         DRM_DEBUG_KMS("force priority a to high\n");
2297                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2298                 }
2299                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2300                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2301                     !evergreen_check_latency_hiding(&wm_low) ||
2302                     (rdev->disp_priority == 2)) {
2303                         DRM_DEBUG_KMS("force priority b to high\n");
2304                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2305                 }
2306
2307                 a.full = dfixed_const(1000);
2308                 b.full = dfixed_const(mode->clock);
2309                 b.full = dfixed_div(b, a);
2310                 c.full = dfixed_const(latency_watermark_a);
2311                 c.full = dfixed_mul(c, b);
2312                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2313                 c.full = dfixed_div(c, a);
2314                 a.full = dfixed_const(16);
2315                 c.full = dfixed_div(c, a);
2316                 priority_a_mark = dfixed_trunc(c);
2317                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2318
2319                 a.full = dfixed_const(1000);
2320                 b.full = dfixed_const(mode->clock);
2321                 b.full = dfixed_div(b, a);
2322                 c.full = dfixed_const(latency_watermark_b);
2323                 c.full = dfixed_mul(c, b);
2324                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2325                 c.full = dfixed_div(c, a);
2326                 a.full = dfixed_const(16);
2327                 c.full = dfixed_div(c, a);
2328                 priority_b_mark = dfixed_trunc(c);
2329                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2330         }
2331
2332         /* select wm A */
2333         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2334         tmp = arb_control3;
2335         tmp &= ~LATENCY_WATERMARK_MASK(3);
2336         tmp |= LATENCY_WATERMARK_MASK(1);
2337         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2338         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2339                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2340                 LATENCY_HIGH_WATERMARK(line_time)));
2341         /* select wm B */
2342         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2343         tmp &= ~LATENCY_WATERMARK_MASK(3);
2344         tmp |= LATENCY_WATERMARK_MASK(2);
2345         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2346         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2347                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2348                 LATENCY_HIGH_WATERMARK(line_time)));
2349         /* restore original selection */
2350         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2351
2352         /* write the priority marks */
2353         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2354         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2355
2356         /* save values for DPM */
2357         radeon_crtc->line_time = line_time;
2358         radeon_crtc->wm_high = latency_watermark_a;
2359         radeon_crtc->wm_low = latency_watermark_b;
2360 }
2361
2362 /**
2363  * evergreen_bandwidth_update - update display watermarks callback.
2364  *
2365  * @rdev: radeon_device pointer
2366  *
2367  * Update the display watermarks based on the requested mode(s)
2368  * (evergreen+).
2369  */
2370 void evergreen_bandwidth_update(struct radeon_device *rdev)
2371 {
2372         struct drm_display_mode *mode0 = NULL;
2373         struct drm_display_mode *mode1 = NULL;
2374         u32 num_heads = 0, lb_size;
2375         int i;
2376
2377         if (!rdev->mode_info.mode_config_initialized)
2378                 return;
2379
2380         radeon_update_display_priority(rdev);
2381
2382         for (i = 0; i < rdev->num_crtc; i++) {
2383                 if (rdev->mode_info.crtcs[i]->base.enabled)
2384                         num_heads++;
2385         }
2386         for (i = 0; i < rdev->num_crtc; i += 2) {
2387                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2388                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2389                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2390                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2391                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2392                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2393         }
2394 }
2395
2396 /**
2397  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2398  *
2399  * @rdev: radeon_device pointer
2400  *
2401  * Wait for the MC (memory controller) to be idle.
2402  * (evergreen+).
2403  * Returns 0 if the MC is idle, -1 if not.
2404  */
2405 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2406 {
2407         unsigned i;
2408         u32 tmp;
2409
2410         for (i = 0; i < rdev->usec_timeout; i++) {
2411                 /* read MC_STATUS */
2412                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2413                 if (!tmp)
2414                         return 0;
2415                 udelay(1);
2416         }
2417         return -1;
2418 }
2419
2420 /*
2421  * GART
2422  */
2423 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2424 {
2425         unsigned i;
2426         u32 tmp;
2427
2428         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2429
2430         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2431         for (i = 0; i < rdev->usec_timeout; i++) {
2432                 /* read MC_STATUS */
2433                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2434                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2435                 if (tmp == 2) {
2436                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2437                         return;
2438                 }
2439                 if (tmp) {
2440                         return;
2441                 }
2442                 udelay(1);
2443         }
2444 }
2445
2446 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2447 {
2448         u32 tmp;
2449         int r;
2450
2451         if (rdev->gart.robj == NULL) {
2452                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2453                 return -EINVAL;
2454         }
2455         r = radeon_gart_table_vram_pin(rdev);
2456         if (r)
2457                 return r;
2458         /* Setup L2 cache */
2459         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2460                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2461                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2462         WREG32(VM_L2_CNTL2, 0);
2463         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2464         /* Setup TLB control */
2465         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2466                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2467                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2468                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2469         if (rdev->flags & RADEON_IS_IGP) {
2470                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2471                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2472                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2473         } else {
2474                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2475                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2476                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2477                 if ((rdev->family == CHIP_JUNIPER) ||
2478                     (rdev->family == CHIP_CYPRESS) ||
2479                     (rdev->family == CHIP_HEMLOCK) ||
2480                     (rdev->family == CHIP_BARTS))
2481                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2482         }
2483         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2484         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2485         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2486         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2487         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2488         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2489         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2490         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2491                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2492         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2493                         (u32)(rdev->dummy_page.addr >> 12));
2494         WREG32(VM_CONTEXT1_CNTL, 0);
2495
2496         evergreen_pcie_gart_tlb_flush(rdev);
2497         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2498                  (unsigned)(rdev->mc.gtt_size >> 20),
2499                  (unsigned long long)rdev->gart.table_addr);
2500         rdev->gart.ready = true;
2501         return 0;
2502 }
2503
2504 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2505 {
2506         u32 tmp;
2507
2508         /* Disable all tables */
2509         WREG32(VM_CONTEXT0_CNTL, 0);
2510         WREG32(VM_CONTEXT1_CNTL, 0);
2511
2512         /* Setup L2 cache */
2513         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2514                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2515         WREG32(VM_L2_CNTL2, 0);
2516         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2517         /* Setup TLB control */
2518         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2519         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2520         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2521         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2522         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2523         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2524         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2525         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2526         radeon_gart_table_vram_unpin(rdev);
2527 }
2528
2529 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2530 {
2531         evergreen_pcie_gart_disable(rdev);
2532         radeon_gart_table_vram_free(rdev);
2533         radeon_gart_fini(rdev);
2534 }
2535
2536
2537 static void evergreen_agp_enable(struct radeon_device *rdev)
2538 {
2539         u32 tmp;
2540
2541         /* Setup L2 cache */
2542         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2543                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2544                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2545         WREG32(VM_L2_CNTL2, 0);
2546         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2547         /* Setup TLB control */
2548         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2549                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2550                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2551                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2552         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2553         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2554         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2555         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2556         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2557         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2558         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2559         WREG32(VM_CONTEXT0_CNTL, 0);
2560         WREG32(VM_CONTEXT1_CNTL, 0);
2561 }
2562
2563 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2564 {
2565         u32 crtc_enabled, tmp, frame_count, blackout;
2566         int i, j;
2567
2568         if (!ASIC_IS_NODCE(rdev)) {
2569                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2570                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2571
2572                 /* disable VGA render */
2573                 WREG32(VGA_RENDER_CONTROL, 0);
2574         }
2575         /* blank the display controllers */
2576         for (i = 0; i < rdev->num_crtc; i++) {
2577                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2578                 if (crtc_enabled) {
2579                         save->crtc_enabled[i] = true;
2580                         if (ASIC_IS_DCE6(rdev)) {
2581                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2582                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2583                                         radeon_wait_for_vblank(rdev, i);
2584                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2585                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2586                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2587                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2588                                 }
2589                         } else {
2590                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2591                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2592                                         radeon_wait_for_vblank(rdev, i);
2593                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2594                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2595                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2596                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2597                                 }
2598                         }
2599                         /* wait for the next frame */
2600                         frame_count = radeon_get_vblank_counter(rdev, i);
2601                         for (j = 0; j < rdev->usec_timeout; j++) {
2602                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2603                                         break;
2604                                 udelay(1);
2605                         }
2606
2607                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2608                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2609                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2610                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2611                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2612                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2613                         save->crtc_enabled[i] = false;
2614                         /* ***** */
2615                 } else {
2616                         save->crtc_enabled[i] = false;
2617                 }
2618         }
2619
2620         radeon_mc_wait_for_idle(rdev);
2621
2622         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2623         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2624                 /* Block CPU access */
2625                 WREG32(BIF_FB_EN, 0);
2626                 /* blackout the MC */
2627                 blackout &= ~BLACKOUT_MODE_MASK;
2628                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2629         }
2630         /* wait for the MC to settle */
2631         udelay(100);
2632
2633         /* lock double buffered regs */
2634         for (i = 0; i < rdev->num_crtc; i++) {
2635                 if (save->crtc_enabled[i]) {
2636                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2637                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2638                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2639                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2640                         }
2641                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2642                         if (!(tmp & 1)) {
2643                                 tmp |= 1;
2644                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2645                         }
2646                 }
2647         }
2648 }
2649
2650 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2651 {
2652         u32 tmp, frame_count;
2653         int i, j;
2654
2655         /* update crtc base addresses */
2656         for (i = 0; i < rdev->num_crtc; i++) {
2657                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2658                        upper_32_bits(rdev->mc.vram_start));
2659                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2660                        upper_32_bits(rdev->mc.vram_start));
2661                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2662                        (u32)rdev->mc.vram_start);
2663                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2664                        (u32)rdev->mc.vram_start);
2665         }
2666
2667         if (!ASIC_IS_NODCE(rdev)) {
2668                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2669                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2670         }
2671
2672         /* unlock regs and wait for update */
2673         for (i = 0; i < rdev->num_crtc; i++) {
2674                 if (save->crtc_enabled[i]) {
2675                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2676                         if ((tmp & 0x7) != 3) {
2677                                 tmp &= ~0x7;
2678                                 tmp |= 0x3;
2679                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2680                         }
2681                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2682                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2683                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2684                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2685                         }
2686                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2687                         if (tmp & 1) {
2688                                 tmp &= ~1;
2689                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2690                         }
2691                         for (j = 0; j < rdev->usec_timeout; j++) {
2692                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2693                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2694                                         break;
2695                                 udelay(1);
2696                         }
2697                 }
2698         }
2699
2700         /* unblackout the MC */
2701         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2702         tmp &= ~BLACKOUT_MODE_MASK;
2703         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2704         /* allow CPU access */
2705         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2706
2707         for (i = 0; i < rdev->num_crtc; i++) {
2708                 if (save->crtc_enabled[i]) {
2709                         if (ASIC_IS_DCE6(rdev)) {
2710                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2711                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2712                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2713                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2714                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2715                         } else {
2716                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2717                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2718                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2719                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2720                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2721                         }
2722                         /* wait for the next frame */
2723                         frame_count = radeon_get_vblank_counter(rdev, i);
2724                         for (j = 0; j < rdev->usec_timeout; j++) {
2725                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2726                                         break;
2727                                 udelay(1);
2728                         }
2729                 }
2730         }
2731         if (!ASIC_IS_NODCE(rdev)) {
2732                 /* Unlock vga access */
2733                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2734                 mdelay(1);
2735                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2736         }
2737 }
2738
2739 void evergreen_mc_program(struct radeon_device *rdev)
2740 {
2741         struct evergreen_mc_save save;
2742         u32 tmp;
2743         int i, j;
2744
2745         /* Initialize HDP */
2746         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2747                 WREG32((0x2c14 + j), 0x00000000);
2748                 WREG32((0x2c18 + j), 0x00000000);
2749                 WREG32((0x2c1c + j), 0x00000000);
2750                 WREG32((0x2c20 + j), 0x00000000);
2751                 WREG32((0x2c24 + j), 0x00000000);
2752         }
2753         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2754
2755         evergreen_mc_stop(rdev, &save);
2756         if (evergreen_mc_wait_for_idle(rdev)) {
2757                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2758         }
2759         /* Lockout access through VGA aperture*/
2760         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2761         /* Update configuration */
2762         if (rdev->flags & RADEON_IS_AGP) {
2763                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2764                         /* VRAM before AGP */
2765                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2766                                 rdev->mc.vram_start >> 12);
2767                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2768                                 rdev->mc.gtt_end >> 12);
2769                 } else {
2770                         /* VRAM after AGP */
2771                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2772                                 rdev->mc.gtt_start >> 12);
2773                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2774                                 rdev->mc.vram_end >> 12);
2775                 }
2776         } else {
2777                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2778                         rdev->mc.vram_start >> 12);
2779                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2780                         rdev->mc.vram_end >> 12);
2781         }
2782         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2783         /* llano/ontario only */
2784         if ((rdev->family == CHIP_PALM) ||
2785             (rdev->family == CHIP_SUMO) ||
2786             (rdev->family == CHIP_SUMO2)) {
2787                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2788                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2789                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2790                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2791         }
2792         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2793         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2794         WREG32(MC_VM_FB_LOCATION, tmp);
2795         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2796         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2797         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2798         if (rdev->flags & RADEON_IS_AGP) {
2799                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2800                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2801                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2802         } else {
2803                 WREG32(MC_VM_AGP_BASE, 0);
2804                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2805                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2806         }
2807         if (evergreen_mc_wait_for_idle(rdev)) {
2808                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2809         }
2810         evergreen_mc_resume(rdev, &save);
2811         /* we need to own VRAM, so turn off the VGA renderer here
2812          * to stop it overwriting our objects */
2813         rv515_vga_render_disable(rdev);
2814 }
2815
2816 /*
2817  * CP.
2818  */
2819 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2820 {
2821         struct radeon_ring *ring = &rdev->ring[ib->ring];
2822         u32 next_rptr;
2823
2824         /* set to DX10/11 mode */
2825         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2826         radeon_ring_write(ring, 1);
2827
2828         if (ring->rptr_save_reg) {
2829                 next_rptr = ring->wptr + 3 + 4;
2830                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2831                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2832                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2833                 radeon_ring_write(ring, next_rptr);
2834         } else if (rdev->wb.enabled) {
2835                 next_rptr = ring->wptr + 5 + 4;
2836                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2837                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2838                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2839                 radeon_ring_write(ring, next_rptr);
2840                 radeon_ring_write(ring, 0);
2841         }
2842
2843         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2844         radeon_ring_write(ring,
2845 #ifdef __BIG_ENDIAN
2846                           (2 << 0) |
2847 #endif
2848                           (ib->gpu_addr & 0xFFFFFFFC));
2849         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2850         radeon_ring_write(ring, ib->length_dw);
2851 }
2852
2853
2854 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2855 {
2856         const __be32 *fw_data;
2857         int i;
2858
2859         if (!rdev->me_fw || !rdev->pfp_fw)
2860                 return -EINVAL;
2861
2862         r700_cp_stop(rdev);
2863         WREG32(CP_RB_CNTL,
2864 #ifdef __BIG_ENDIAN
2865                BUF_SWAP_32BIT |
2866 #endif
2867                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2868
2869         fw_data = (const __be32 *)rdev->pfp_fw->data;
2870         WREG32(CP_PFP_UCODE_ADDR, 0);
2871         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2872                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2873         WREG32(CP_PFP_UCODE_ADDR, 0);
2874
2875         fw_data = (const __be32 *)rdev->me_fw->data;
2876         WREG32(CP_ME_RAM_WADDR, 0);
2877         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2878                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2879
2880         WREG32(CP_PFP_UCODE_ADDR, 0);
2881         WREG32(CP_ME_RAM_WADDR, 0);
2882         WREG32(CP_ME_RAM_RADDR, 0);
2883         return 0;
2884 }
2885
2886 static int evergreen_cp_start(struct radeon_device *rdev)
2887 {
2888         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2889         int r, i;
2890         uint32_t cp_me;
2891
2892         r = radeon_ring_lock(rdev, ring, 7);
2893         if (r) {
2894                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2895                 return r;
2896         }
2897         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2898         radeon_ring_write(ring, 0x1);
2899         radeon_ring_write(ring, 0x0);
2900         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2901         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2902         radeon_ring_write(ring, 0);
2903         radeon_ring_write(ring, 0);
2904         radeon_ring_unlock_commit(rdev, ring, false);
2905
2906         cp_me = 0xff;
2907         WREG32(CP_ME_CNTL, cp_me);
2908
2909         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2910         if (r) {
2911                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2912                 return r;
2913         }
2914
2915         /* setup clear context state */
2916         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2917         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2918
2919         for (i = 0; i < evergreen_default_size; i++)
2920                 radeon_ring_write(ring, evergreen_default_state[i]);
2921
2922         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2923         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2924
2925         /* set clear context state */
2926         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2927         radeon_ring_write(ring, 0);
2928
2929         /* SQ_VTX_BASE_VTX_LOC */
2930         radeon_ring_write(ring, 0xc0026f00);
2931         radeon_ring_write(ring, 0x00000000);
2932         radeon_ring_write(ring, 0x00000000);
2933         radeon_ring_write(ring, 0x00000000);
2934
2935         /* Clear consts */
2936         radeon_ring_write(ring, 0xc0036f00);
2937         radeon_ring_write(ring, 0x00000bc4);
2938         radeon_ring_write(ring, 0xffffffff);
2939         radeon_ring_write(ring, 0xffffffff);
2940         radeon_ring_write(ring, 0xffffffff);
2941
2942         radeon_ring_write(ring, 0xc0026900);
2943         radeon_ring_write(ring, 0x00000316);
2944         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2945         radeon_ring_write(ring, 0x00000010); /*  */
2946
2947         radeon_ring_unlock_commit(rdev, ring, false);
2948
2949         return 0;
2950 }
2951
2952 static int evergreen_cp_resume(struct radeon_device *rdev)
2953 {
2954         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2955         u32 tmp;
2956         u32 rb_bufsz;
2957         int r;
2958
2959         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2960         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2961                                  SOFT_RESET_PA |
2962                                  SOFT_RESET_SH |
2963                                  SOFT_RESET_VGT |
2964                                  SOFT_RESET_SPI |
2965                                  SOFT_RESET_SX));
2966         RREG32(GRBM_SOFT_RESET);
2967         mdelay(15);
2968         WREG32(GRBM_SOFT_RESET, 0);
2969         RREG32(GRBM_SOFT_RESET);
2970
2971         /* Set ring buffer size */
2972         rb_bufsz = order_base_2(ring->ring_size / 8);
2973         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2974 #ifdef __BIG_ENDIAN
2975         tmp |= BUF_SWAP_32BIT;
2976 #endif
2977         WREG32(CP_RB_CNTL, tmp);
2978         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2979         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2980
2981         /* Set the write pointer delay */
2982         WREG32(CP_RB_WPTR_DELAY, 0);
2983
2984         /* Initialize the ring buffer's read and write pointers */
2985         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2986         WREG32(CP_RB_RPTR_WR, 0);
2987         ring->wptr = 0;
2988         WREG32(CP_RB_WPTR, ring->wptr);
2989
2990         /* set the wb address whether it's enabled or not */
2991         WREG32(CP_RB_RPTR_ADDR,
2992                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2993         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2994         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2995
2996         if (rdev->wb.enabled)
2997                 WREG32(SCRATCH_UMSK, 0xff);
2998         else {
2999                 tmp |= RB_NO_UPDATE;
3000                 WREG32(SCRATCH_UMSK, 0);
3001         }
3002
3003         mdelay(1);
3004         WREG32(CP_RB_CNTL, tmp);
3005
3006         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3007         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3008
3009         evergreen_cp_start(rdev);
3010         ring->ready = true;
3011         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3012         if (r) {
3013                 ring->ready = false;
3014                 return r;
3015         }
3016         return 0;
3017 }
3018
3019 /*
3020  * Core functions
3021  */
3022 static void evergreen_gpu_init(struct radeon_device *rdev)
3023 {
3024         u32 gb_addr_config;
3025         u32 mc_shared_chmap, mc_arb_ramcfg;
3026         u32 sx_debug_1;
3027         u32 smx_dc_ctl0;
3028         u32 sq_config;
3029         u32 sq_lds_resource_mgmt;
3030         u32 sq_gpr_resource_mgmt_1;
3031         u32 sq_gpr_resource_mgmt_2;
3032         u32 sq_gpr_resource_mgmt_3;
3033         u32 sq_thread_resource_mgmt;
3034         u32 sq_thread_resource_mgmt_2;
3035         u32 sq_stack_resource_mgmt_1;
3036         u32 sq_stack_resource_mgmt_2;
3037         u32 sq_stack_resource_mgmt_3;
3038         u32 vgt_cache_invalidation;
3039         u32 hdp_host_path_cntl, tmp;
3040         u32 disabled_rb_mask;
3041         int i, j, ps_thread_count;
3042
3043         switch (rdev->family) {
3044         case CHIP_CYPRESS:
3045         case CHIP_HEMLOCK:
3046                 rdev->config.evergreen.num_ses = 2;
3047                 rdev->config.evergreen.max_pipes = 4;
3048                 rdev->config.evergreen.max_tile_pipes = 8;
3049                 rdev->config.evergreen.max_simds = 10;
3050                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3051                 rdev->config.evergreen.max_gprs = 256;
3052                 rdev->config.evergreen.max_threads = 248;
3053                 rdev->config.evergreen.max_gs_threads = 32;
3054                 rdev->config.evergreen.max_stack_entries = 512;
3055                 rdev->config.evergreen.sx_num_of_sets = 4;
3056                 rdev->config.evergreen.sx_max_export_size = 256;
3057                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3058                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3059                 rdev->config.evergreen.max_hw_contexts = 8;
3060                 rdev->config.evergreen.sq_num_cf_insts = 2;
3061
3062                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3063                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3064                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3065                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3066                 break;
3067         case CHIP_JUNIPER:
3068                 rdev->config.evergreen.num_ses = 1;
3069                 rdev->config.evergreen.max_pipes = 4;
3070                 rdev->config.evergreen.max_tile_pipes = 4;
3071                 rdev->config.evergreen.max_simds = 10;
3072                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3073                 rdev->config.evergreen.max_gprs = 256;
3074                 rdev->config.evergreen.max_threads = 248;
3075                 rdev->config.evergreen.max_gs_threads = 32;
3076                 rdev->config.evergreen.max_stack_entries = 512;
3077                 rdev->config.evergreen.sx_num_of_sets = 4;
3078                 rdev->config.evergreen.sx_max_export_size = 256;
3079                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3080                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3081                 rdev->config.evergreen.max_hw_contexts = 8;
3082                 rdev->config.evergreen.sq_num_cf_insts = 2;
3083
3084                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3085                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3086                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3087                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3088                 break;
3089         case CHIP_REDWOOD:
3090                 rdev->config.evergreen.num_ses = 1;
3091                 rdev->config.evergreen.max_pipes = 4;
3092                 rdev->config.evergreen.max_tile_pipes = 4;
3093                 rdev->config.evergreen.max_simds = 5;
3094                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3095                 rdev->config.evergreen.max_gprs = 256;
3096                 rdev->config.evergreen.max_threads = 248;
3097                 rdev->config.evergreen.max_gs_threads = 32;
3098                 rdev->config.evergreen.max_stack_entries = 256;
3099                 rdev->config.evergreen.sx_num_of_sets = 4;
3100                 rdev->config.evergreen.sx_max_export_size = 256;
3101                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3102                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3103                 rdev->config.evergreen.max_hw_contexts = 8;
3104                 rdev->config.evergreen.sq_num_cf_insts = 2;
3105
3106                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3107                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3108                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3109                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3110                 break;
3111         case CHIP_CEDAR:
3112         default:
3113                 rdev->config.evergreen.num_ses = 1;
3114                 rdev->config.evergreen.max_pipes = 2;
3115                 rdev->config.evergreen.max_tile_pipes = 2;
3116                 rdev->config.evergreen.max_simds = 2;
3117                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3118                 rdev->config.evergreen.max_gprs = 256;
3119                 rdev->config.evergreen.max_threads = 192;
3120                 rdev->config.evergreen.max_gs_threads = 16;
3121                 rdev->config.evergreen.max_stack_entries = 256;
3122                 rdev->config.evergreen.sx_num_of_sets = 4;
3123                 rdev->config.evergreen.sx_max_export_size = 128;
3124                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3125                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3126                 rdev->config.evergreen.max_hw_contexts = 4;
3127                 rdev->config.evergreen.sq_num_cf_insts = 1;
3128
3129                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3130                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3131                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3132                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3133                 break;
3134         case CHIP_PALM:
3135                 rdev->config.evergreen.num_ses = 1;
3136                 rdev->config.evergreen.max_pipes = 2;
3137                 rdev->config.evergreen.max_tile_pipes = 2;
3138                 rdev->config.evergreen.max_simds = 2;
3139                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3140                 rdev->config.evergreen.max_gprs = 256;
3141                 rdev->config.evergreen.max_threads = 192;
3142                 rdev->config.evergreen.max_gs_threads = 16;
3143                 rdev->config.evergreen.max_stack_entries = 256;
3144                 rdev->config.evergreen.sx_num_of_sets = 4;
3145                 rdev->config.evergreen.sx_max_export_size = 128;
3146                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3147                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3148                 rdev->config.evergreen.max_hw_contexts = 4;
3149                 rdev->config.evergreen.sq_num_cf_insts = 1;
3150
3151                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3152                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3153                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3154                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3155                 break;
3156         case CHIP_SUMO:
3157                 rdev->config.evergreen.num_ses = 1;
3158                 rdev->config.evergreen.max_pipes = 4;
3159                 rdev->config.evergreen.max_tile_pipes = 4;
3160                 if (rdev->pdev->device == 0x9648)
3161                         rdev->config.evergreen.max_simds = 3;
3162                 else if ((rdev->pdev->device == 0x9647) ||
3163                          (rdev->pdev->device == 0x964a))
3164                         rdev->config.evergreen.max_simds = 4;
3165                 else
3166                         rdev->config.evergreen.max_simds = 5;
3167                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3168                 rdev->config.evergreen.max_gprs = 256;
3169                 rdev->config.evergreen.max_threads = 248;
3170                 rdev->config.evergreen.max_gs_threads = 32;
3171                 rdev->config.evergreen.max_stack_entries = 256;
3172                 rdev->config.evergreen.sx_num_of_sets = 4;
3173                 rdev->config.evergreen.sx_max_export_size = 256;
3174                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3175                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3176                 rdev->config.evergreen.max_hw_contexts = 8;
3177                 rdev->config.evergreen.sq_num_cf_insts = 2;
3178
3179                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3180                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3181                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3182                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3183                 break;
3184         case CHIP_SUMO2:
3185                 rdev->config.evergreen.num_ses = 1;
3186                 rdev->config.evergreen.max_pipes = 4;
3187                 rdev->config.evergreen.max_tile_pipes = 4;
3188                 rdev->config.evergreen.max_simds = 2;
3189                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3190                 rdev->config.evergreen.max_gprs = 256;
3191                 rdev->config.evergreen.max_threads = 248;
3192                 rdev->config.evergreen.max_gs_threads = 32;
3193                 rdev->config.evergreen.max_stack_entries = 512;
3194                 rdev->config.evergreen.sx_num_of_sets = 4;
3195                 rdev->config.evergreen.sx_max_export_size = 256;
3196                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3197                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3198                 rdev->config.evergreen.max_hw_contexts = 4;
3199                 rdev->config.evergreen.sq_num_cf_insts = 2;
3200
3201                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3202                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3203                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3204                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3205                 break;
3206         case CHIP_BARTS:
3207                 rdev->config.evergreen.num_ses = 2;
3208                 rdev->config.evergreen.max_pipes = 4;
3209                 rdev->config.evergreen.max_tile_pipes = 8;
3210                 rdev->config.evergreen.max_simds = 7;
3211                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3212                 rdev->config.evergreen.max_gprs = 256;
3213                 rdev->config.evergreen.max_threads = 248;
3214                 rdev->config.evergreen.max_gs_threads = 32;
3215                 rdev->config.evergreen.max_stack_entries = 512;
3216                 rdev->config.evergreen.sx_num_of_sets = 4;
3217                 rdev->config.evergreen.sx_max_export_size = 256;
3218                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3219                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3220                 rdev->config.evergreen.max_hw_contexts = 8;
3221                 rdev->config.evergreen.sq_num_cf_insts = 2;
3222
3223                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3224                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3225                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3226                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3227                 break;
3228         case CHIP_TURKS:
3229                 rdev->config.evergreen.num_ses = 1;
3230                 rdev->config.evergreen.max_pipes = 4;
3231                 rdev->config.evergreen.max_tile_pipes = 4;
3232                 rdev->config.evergreen.max_simds = 6;
3233                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3234                 rdev->config.evergreen.max_gprs = 256;
3235                 rdev->config.evergreen.max_threads = 248;
3236                 rdev->config.evergreen.max_gs_threads = 32;
3237                 rdev->config.evergreen.max_stack_entries = 256;
3238                 rdev->config.evergreen.sx_num_of_sets = 4;
3239                 rdev->config.evergreen.sx_max_export_size = 256;
3240                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3241                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3242                 rdev->config.evergreen.max_hw_contexts = 8;
3243                 rdev->config.evergreen.sq_num_cf_insts = 2;
3244
3245                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3246                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3247                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3248                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3249                 break;
3250         case CHIP_CAICOS:
3251                 rdev->config.evergreen.num_ses = 1;
3252                 rdev->config.evergreen.max_pipes = 2;
3253                 rdev->config.evergreen.max_tile_pipes = 2;
3254                 rdev->config.evergreen.max_simds = 2;
3255                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3256                 rdev->config.evergreen.max_gprs = 256;
3257                 rdev->config.evergreen.max_threads = 192;
3258                 rdev->config.evergreen.max_gs_threads = 16;
3259                 rdev->config.evergreen.max_stack_entries = 256;
3260                 rdev->config.evergreen.sx_num_of_sets = 4;
3261                 rdev->config.evergreen.sx_max_export_size = 128;
3262                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3263                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3264                 rdev->config.evergreen.max_hw_contexts = 4;
3265                 rdev->config.evergreen.sq_num_cf_insts = 1;
3266
3267                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3268                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3269                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3270                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3271                 break;
3272         }
3273
3274         /* Initialize HDP */
3275         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3276                 WREG32((0x2c14 + j), 0x00000000);
3277                 WREG32((0x2c18 + j), 0x00000000);
3278                 WREG32((0x2c1c + j), 0x00000000);
3279                 WREG32((0x2c20 + j), 0x00000000);
3280                 WREG32((0x2c24 + j), 0x00000000);
3281         }
3282
3283         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3284         WREG32(SRBM_INT_CNTL, 0x1);
3285         WREG32(SRBM_INT_ACK, 0x1);
3286
3287         evergreen_fix_pci_max_read_req_size(rdev);
3288
3289         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3290         if ((rdev->family == CHIP_PALM) ||
3291             (rdev->family == CHIP_SUMO) ||
3292             (rdev->family == CHIP_SUMO2))
3293                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3294         else
3295                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3296
3297         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3298          * not have bank info, so create a custom tiling dword.
3299          * bits 3:0   num_pipes
3300          * bits 7:4   num_banks
3301          * bits 11:8  group_size
3302          * bits 15:12 row_size
3303          */
3304         rdev->config.evergreen.tile_config = 0;
3305         switch (rdev->config.evergreen.max_tile_pipes) {
3306         case 1:
3307         default:
3308                 rdev->config.evergreen.tile_config |= (0 << 0);
3309                 break;
3310         case 2:
3311                 rdev->config.evergreen.tile_config |= (1 << 0);
3312                 break;
3313         case 4:
3314                 rdev->config.evergreen.tile_config |= (2 << 0);
3315                 break;
3316         case 8:
3317                 rdev->config.evergreen.tile_config |= (3 << 0);
3318                 break;
3319         }
3320         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3321         if (rdev->flags & RADEON_IS_IGP)
3322                 rdev->config.evergreen.tile_config |= 1 << 4;
3323         else {
3324                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3325                 case 0: /* four banks */
3326                         rdev->config.evergreen.tile_config |= 0 << 4;
3327                         break;
3328                 case 1: /* eight banks */
3329                         rdev->config.evergreen.tile_config |= 1 << 4;
3330                         break;
3331                 case 2: /* sixteen banks */
3332                 default:
3333                         rdev->config.evergreen.tile_config |= 2 << 4;
3334                         break;
3335                 }
3336         }
3337         rdev->config.evergreen.tile_config |= 0 << 8;
3338         rdev->config.evergreen.tile_config |=
3339                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3340
3341         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3342                 u32 efuse_straps_4;
3343                 u32 efuse_straps_3;
3344
3345                 efuse_straps_4 = RREG32_RCU(0x204);
3346                 efuse_straps_3 = RREG32_RCU(0x203);
3347                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3348                       ((efuse_straps_3 & 0xf0000000) >> 28));
3349         } else {
3350                 tmp = 0;
3351                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3352                         u32 rb_disable_bitmap;
3353
3354                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3355                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3356                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3357                         tmp <<= 4;
3358                         tmp |= rb_disable_bitmap;
3359                 }
3360         }
3361         /* enabled rb are just the one not disabled :) */
3362         disabled_rb_mask = tmp;
3363         tmp = 0;
3364         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3365                 tmp |= (1 << i);
3366         /* if all the backends are disabled, fix it up here */
3367         if ((disabled_rb_mask & tmp) == tmp) {
3368                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3369                         disabled_rb_mask &= ~(1 << i);
3370         }
3371
3372         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3373                 u32 simd_disable_bitmap;
3374
3375                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3376                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3377                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3378                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3379                 tmp <<= 16;
3380                 tmp |= simd_disable_bitmap;
3381         }
3382         rdev->config.evergreen.active_simds = hweight32(~tmp);
3383
3384         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3385         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3386
3387         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3388         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3389         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3390         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3391         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3392         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3393         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3394
3395         if ((rdev->config.evergreen.max_backends == 1) &&
3396             (rdev->flags & RADEON_IS_IGP)) {
3397                 if ((disabled_rb_mask & 3) == 1) {
3398                         /* RB0 disabled, RB1 enabled */
3399                         tmp = 0x11111111;
3400                 } else {
3401                         /* RB1 disabled, RB0 enabled */
3402                         tmp = 0x00000000;
3403                 }
3404         } else {
3405                 tmp = gb_addr_config & NUM_PIPES_MASK;
3406                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3407                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3408         }
3409         WREG32(GB_BACKEND_MAP, tmp);
3410
3411         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3412         WREG32(CGTS_TCC_DISABLE, 0);
3413         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3414         WREG32(CGTS_USER_TCC_DISABLE, 0);
3415
3416         /* set HW defaults for 3D engine */
3417         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3418                                      ROQ_IB2_START(0x2b)));
3419
3420         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3421
3422         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3423                              SYNC_GRADIENT |
3424                              SYNC_WALKER |
3425                              SYNC_ALIGNER));
3426
3427         sx_debug_1 = RREG32(SX_DEBUG_1);
3428         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3429         WREG32(SX_DEBUG_1, sx_debug_1);
3430
3431
3432         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3433         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3434         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3435         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3436
3437         if (rdev->family <= CHIP_SUMO2)
3438                 WREG32(SMX_SAR_CTL0, 0x00010000);
3439
3440         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3441                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3442                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3443
3444         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3445                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3446                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3447
3448         WREG32(VGT_NUM_INSTANCES, 1);
3449         WREG32(SPI_CONFIG_CNTL, 0);
3450         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3451         WREG32(CP_PERFMON_CNTL, 0);
3452
3453         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3454                                   FETCH_FIFO_HIWATER(0x4) |
3455                                   DONE_FIFO_HIWATER(0xe0) |
3456                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3457
3458         sq_config = RREG32(SQ_CONFIG);
3459         sq_config &= ~(PS_PRIO(3) |
3460                        VS_PRIO(3) |
3461                        GS_PRIO(3) |
3462                        ES_PRIO(3));
3463         sq_config |= (VC_ENABLE |
3464                       EXPORT_SRC_C |
3465                       PS_PRIO(0) |
3466                       VS_PRIO(1) |
3467                       GS_PRIO(2) |
3468                       ES_PRIO(3));
3469
3470         switch (rdev->family) {
3471         case CHIP_CEDAR:
3472         case CHIP_PALM:
3473         case CHIP_SUMO:
3474         case CHIP_SUMO2:
3475         case CHIP_CAICOS:
3476                 /* no vertex cache */
3477                 sq_config &= ~VC_ENABLE;
3478                 break;
3479         default:
3480                 break;
3481         }
3482
3483         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3484
3485         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3486         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3487         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3488         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3489         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3490         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3491         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3492
3493         switch (rdev->family) {
3494         case CHIP_CEDAR:
3495         case CHIP_PALM:
3496         case CHIP_SUMO:
3497         case CHIP_SUMO2:
3498                 ps_thread_count = 96;
3499                 break;
3500         default:
3501                 ps_thread_count = 128;
3502                 break;
3503         }
3504
3505         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3506         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3507         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3508         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3509         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3510         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3511
3512         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3513         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3514         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3515         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3516         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3517         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3518
3519         WREG32(SQ_CONFIG, sq_config);
3520         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3521         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3522         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3523         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3524         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3525         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3526         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3527         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3528         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3529         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3530
3531         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3532                                           FORCE_EOV_MAX_REZ_CNT(255)));
3533
3534         switch (rdev->family) {
3535         case CHIP_CEDAR:
3536         case CHIP_PALM:
3537         case CHIP_SUMO:
3538         case CHIP_SUMO2:
3539         case CHIP_CAICOS:
3540                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3541                 break;
3542         default:
3543                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3544                 break;
3545         }
3546         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3547         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3548
3549         WREG32(VGT_GS_VERTEX_REUSE, 16);
3550         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3551         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3552
3553         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3554         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3555
3556         WREG32(CB_PERF_CTR0_SEL_0, 0);
3557         WREG32(CB_PERF_CTR0_SEL_1, 0);
3558         WREG32(CB_PERF_CTR1_SEL_0, 0);
3559         WREG32(CB_PERF_CTR1_SEL_1, 0);
3560         WREG32(CB_PERF_CTR2_SEL_0, 0);
3561         WREG32(CB_PERF_CTR2_SEL_1, 0);
3562         WREG32(CB_PERF_CTR3_SEL_0, 0);
3563         WREG32(CB_PERF_CTR3_SEL_1, 0);
3564
3565         /* clear render buffer base addresses */
3566         WREG32(CB_COLOR0_BASE, 0);
3567         WREG32(CB_COLOR1_BASE, 0);
3568         WREG32(CB_COLOR2_BASE, 0);
3569         WREG32(CB_COLOR3_BASE, 0);
3570         WREG32(CB_COLOR4_BASE, 0);
3571         WREG32(CB_COLOR5_BASE, 0);
3572         WREG32(CB_COLOR6_BASE, 0);
3573         WREG32(CB_COLOR7_BASE, 0);
3574         WREG32(CB_COLOR8_BASE, 0);
3575         WREG32(CB_COLOR9_BASE, 0);
3576         WREG32(CB_COLOR10_BASE, 0);
3577         WREG32(CB_COLOR11_BASE, 0);
3578
3579         /* set the shader const cache sizes to 0 */
3580         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3581                 WREG32(i, 0);
3582         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3583                 WREG32(i, 0);
3584
3585         tmp = RREG32(HDP_MISC_CNTL);
3586         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3587         WREG32(HDP_MISC_CNTL, tmp);
3588
3589         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3590         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3591
3592         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3593
3594         udelay(50);
3595
3596 }
3597
3598 int evergreen_mc_init(struct radeon_device *rdev)
3599 {
3600         u32 tmp;
3601         int chansize, numchan;
3602
3603         /* Get VRAM informations */
3604         rdev->mc.vram_is_ddr = true;
3605         if ((rdev->family == CHIP_PALM) ||
3606             (rdev->family == CHIP_SUMO) ||
3607             (rdev->family == CHIP_SUMO2))
3608                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3609         else
3610                 tmp = RREG32(MC_ARB_RAMCFG);
3611         if (tmp & CHANSIZE_OVERRIDE) {
3612                 chansize = 16;
3613         } else if (tmp & CHANSIZE_MASK) {
3614                 chansize = 64;
3615         } else {
3616                 chansize = 32;
3617         }
3618         tmp = RREG32(MC_SHARED_CHMAP);
3619         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3620         case 0:
3621         default:
3622                 numchan = 1;
3623                 break;
3624         case 1:
3625                 numchan = 2;
3626                 break;
3627         case 2:
3628                 numchan = 4;
3629                 break;
3630         case 3:
3631                 numchan = 8;
3632                 break;
3633         }
3634         rdev->mc.vram_width = numchan * chansize;
3635         /* Could aper size report 0 ? */
3636         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3637         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3638         /* Setup GPU memory space */
3639         if ((rdev->family == CHIP_PALM) ||
3640             (rdev->family == CHIP_SUMO) ||
3641             (rdev->family == CHIP_SUMO2)) {
3642                 /* size in bytes on fusion */
3643                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3644                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3645         } else {
3646                 /* size in MB on evergreen/cayman/tn */
3647                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3648                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3649         }
3650         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3651         r700_vram_gtt_location(rdev, &rdev->mc);
3652         radeon_update_bandwidth_info(rdev);
3653
3654         return 0;
3655 }
3656
3657 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3658 {
3659         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3660                 RREG32(GRBM_STATUS));
3661         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3662                 RREG32(GRBM_STATUS_SE0));
3663         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3664                 RREG32(GRBM_STATUS_SE1));
3665         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3666                 RREG32(SRBM_STATUS));
3667         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3668                 RREG32(SRBM_STATUS2));
3669         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3670                 RREG32(CP_STALLED_STAT1));
3671         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3672                 RREG32(CP_STALLED_STAT2));
3673         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3674                 RREG32(CP_BUSY_STAT));
3675         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3676                 RREG32(CP_STAT));
3677         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3678                 RREG32(DMA_STATUS_REG));
3679         if (rdev->family >= CHIP_CAYMAN) {
3680                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3681                          RREG32(DMA_STATUS_REG + 0x800));
3682         }
3683 }
3684
3685 bool evergreen_is_display_hung(struct radeon_device *rdev)
3686 {
3687         u32 crtc_hung = 0;
3688         u32 crtc_status[6];
3689         u32 i, j, tmp;
3690
3691         for (i = 0; i < rdev->num_crtc; i++) {
3692                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3693                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3694                         crtc_hung |= (1 << i);
3695                 }
3696         }
3697
3698         for (j = 0; j < 10; j++) {
3699                 for (i = 0; i < rdev->num_crtc; i++) {
3700                         if (crtc_hung & (1 << i)) {
3701                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3702                                 if (tmp != crtc_status[i])
3703                                         crtc_hung &= ~(1 << i);
3704                         }
3705                 }
3706                 if (crtc_hung == 0)
3707                         return false;
3708                 udelay(100);
3709         }
3710
3711         return true;
3712 }
3713
3714 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3715 {
3716         u32 reset_mask = 0;
3717         u32 tmp;
3718
3719         /* GRBM_STATUS */
3720         tmp = RREG32(GRBM_STATUS);
3721         if (tmp & (PA_BUSY | SC_BUSY |
3722                    SH_BUSY | SX_BUSY |
3723                    TA_BUSY | VGT_BUSY |
3724                    DB_BUSY | CB_BUSY |
3725                    SPI_BUSY | VGT_BUSY_NO_DMA))
3726                 reset_mask |= RADEON_RESET_GFX;
3727
3728         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3729                    CP_BUSY | CP_COHERENCY_BUSY))
3730                 reset_mask |= RADEON_RESET_CP;
3731
3732         if (tmp & GRBM_EE_BUSY)
3733                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3734
3735         /* DMA_STATUS_REG */
3736         tmp = RREG32(DMA_STATUS_REG);
3737         if (!(tmp & DMA_IDLE))
3738                 reset_mask |= RADEON_RESET_DMA;
3739
3740         /* SRBM_STATUS2 */
3741         tmp = RREG32(SRBM_STATUS2);
3742         if (tmp & DMA_BUSY)
3743                 reset_mask |= RADEON_RESET_DMA;
3744
3745         /* SRBM_STATUS */
3746         tmp = RREG32(SRBM_STATUS);
3747         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3748                 reset_mask |= RADEON_RESET_RLC;
3749
3750         if (tmp & IH_BUSY)
3751                 reset_mask |= RADEON_RESET_IH;
3752
3753         if (tmp & SEM_BUSY)
3754                 reset_mask |= RADEON_RESET_SEM;
3755
3756         if (tmp & GRBM_RQ_PENDING)
3757                 reset_mask |= RADEON_RESET_GRBM;
3758
3759         if (tmp & VMC_BUSY)
3760                 reset_mask |= RADEON_RESET_VMC;
3761
3762         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3763                    MCC_BUSY | MCD_BUSY))
3764                 reset_mask |= RADEON_RESET_MC;
3765
3766         if (evergreen_is_display_hung(rdev))
3767                 reset_mask |= RADEON_RESET_DISPLAY;
3768
3769         /* VM_L2_STATUS */
3770         tmp = RREG32(VM_L2_STATUS);
3771         if (tmp & L2_BUSY)
3772                 reset_mask |= RADEON_RESET_VMC;
3773
3774         /* Skip MC reset as it's mostly likely not hung, just busy */
3775         if (reset_mask & RADEON_RESET_MC) {
3776                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3777                 reset_mask &= ~RADEON_RESET_MC;
3778         }
3779
3780         return reset_mask;
3781 }
3782
3783 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3784 {
3785         struct evergreen_mc_save save;
3786         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3787         u32 tmp;
3788
3789         if (reset_mask == 0)
3790                 return;
3791
3792         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3793
3794         evergreen_print_gpu_status_regs(rdev);
3795
3796         /* Disable CP parsing/prefetching */
3797         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3798
3799         if (reset_mask & RADEON_RESET_DMA) {
3800                 /* Disable DMA */
3801                 tmp = RREG32(DMA_RB_CNTL);
3802                 tmp &= ~DMA_RB_ENABLE;
3803                 WREG32(DMA_RB_CNTL, tmp);
3804         }
3805
3806         udelay(50);
3807
3808         evergreen_mc_stop(rdev, &save);
3809         if (evergreen_mc_wait_for_idle(rdev)) {
3810                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3811         }
3812
3813         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3814                 grbm_soft_reset |= SOFT_RESET_DB |
3815                         SOFT_RESET_CB |
3816                         SOFT_RESET_PA |
3817                         SOFT_RESET_SC |
3818                         SOFT_RESET_SPI |
3819                         SOFT_RESET_SX |
3820                         SOFT_RESET_SH |
3821                         SOFT_RESET_TC |
3822                         SOFT_RESET_TA |
3823                         SOFT_RESET_VC |
3824                         SOFT_RESET_VGT;
3825         }
3826
3827         if (reset_mask & RADEON_RESET_CP) {
3828                 grbm_soft_reset |= SOFT_RESET_CP |
3829                         SOFT_RESET_VGT;
3830
3831                 srbm_soft_reset |= SOFT_RESET_GRBM;
3832         }
3833
3834         if (reset_mask & RADEON_RESET_DMA)
3835                 srbm_soft_reset |= SOFT_RESET_DMA;
3836
3837         if (reset_mask & RADEON_RESET_DISPLAY)
3838                 srbm_soft_reset |= SOFT_RESET_DC;
3839
3840         if (reset_mask & RADEON_RESET_RLC)
3841                 srbm_soft_reset |= SOFT_RESET_RLC;
3842
3843         if (reset_mask & RADEON_RESET_SEM)
3844                 srbm_soft_reset |= SOFT_RESET_SEM;
3845
3846         if (reset_mask & RADEON_RESET_IH)
3847                 srbm_soft_reset |= SOFT_RESET_IH;
3848
3849         if (reset_mask & RADEON_RESET_GRBM)
3850                 srbm_soft_reset |= SOFT_RESET_GRBM;
3851
3852         if (reset_mask & RADEON_RESET_VMC)
3853                 srbm_soft_reset |= SOFT_RESET_VMC;
3854
3855         if (!(rdev->flags & RADEON_IS_IGP)) {
3856                 if (reset_mask & RADEON_RESET_MC)
3857                         srbm_soft_reset |= SOFT_RESET_MC;
3858         }
3859
3860         if (grbm_soft_reset) {
3861                 tmp = RREG32(GRBM_SOFT_RESET);
3862                 tmp |= grbm_soft_reset;
3863                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3864                 WREG32(GRBM_SOFT_RESET, tmp);
3865                 tmp = RREG32(GRBM_SOFT_RESET);
3866
3867                 udelay(50);
3868
3869                 tmp &= ~grbm_soft_reset;
3870                 WREG32(GRBM_SOFT_RESET, tmp);
3871                 tmp = RREG32(GRBM_SOFT_RESET);
3872         }
3873
3874         if (srbm_soft_reset) {
3875                 tmp = RREG32(SRBM_SOFT_RESET);
3876                 tmp |= srbm_soft_reset;
3877                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3878                 WREG32(SRBM_SOFT_RESET, tmp);
3879                 tmp = RREG32(SRBM_SOFT_RESET);
3880
3881                 udelay(50);
3882
3883                 tmp &= ~srbm_soft_reset;
3884                 WREG32(SRBM_SOFT_RESET, tmp);
3885                 tmp = RREG32(SRBM_SOFT_RESET);
3886         }
3887
3888         /* Wait a little for things to settle down */
3889         udelay(50);
3890
3891         evergreen_mc_resume(rdev, &save);
3892         udelay(50);
3893
3894         evergreen_print_gpu_status_regs(rdev);
3895 }
3896
3897 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3898 {
3899         struct evergreen_mc_save save;
3900         u32 tmp, i;
3901
3902         dev_info(rdev->dev, "GPU pci config reset\n");
3903
3904         /* disable dpm? */
3905
3906         /* Disable CP parsing/prefetching */
3907         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3908         udelay(50);
3909         /* Disable DMA */
3910         tmp = RREG32(DMA_RB_CNTL);
3911         tmp &= ~DMA_RB_ENABLE;
3912         WREG32(DMA_RB_CNTL, tmp);
3913         /* XXX other engines? */
3914
3915         /* halt the rlc */
3916         r600_rlc_stop(rdev);
3917
3918         udelay(50);
3919
3920         /* set mclk/sclk to bypass */
3921         rv770_set_clk_bypass_mode(rdev);
3922         /* disable BM */
3923         pci_clear_master(rdev->pdev);
3924         /* disable mem access */
3925         evergreen_mc_stop(rdev, &save);
3926         if (evergreen_mc_wait_for_idle(rdev)) {
3927                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3928         }
3929         /* reset */
3930         radeon_pci_config_reset(rdev);
3931         /* wait for asic to come out of reset */
3932         for (i = 0; i < rdev->usec_timeout; i++) {
3933                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3934                         break;
3935                 udelay(1);
3936         }
3937 }
3938
3939 int evergreen_asic_reset(struct radeon_device *rdev)
3940 {
3941         u32 reset_mask;
3942
3943         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3944
3945         if (reset_mask)
3946                 r600_set_bios_scratch_engine_hung(rdev, true);
3947
3948         /* try soft reset */
3949         evergreen_gpu_soft_reset(rdev, reset_mask);
3950
3951         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3952
3953         /* try pci config reset */
3954         if (reset_mask && radeon_hard_reset)
3955                 evergreen_gpu_pci_config_reset(rdev);
3956
3957         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3958
3959         if (!reset_mask)
3960                 r600_set_bios_scratch_engine_hung(rdev, false);
3961
3962         return 0;
3963 }
3964
3965 /**
3966  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3967  *
3968  * @rdev: radeon_device pointer
3969  * @ring: radeon_ring structure holding ring information
3970  *
3971  * Check if the GFX engine is locked up.
3972  * Returns true if the engine appears to be locked up, false if not.
3973  */
3974 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3975 {
3976         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3977
3978         if (!(reset_mask & (RADEON_RESET_GFX |
3979                             RADEON_RESET_COMPUTE |
3980                             RADEON_RESET_CP))) {
3981                 radeon_ring_lockup_update(rdev, ring);
3982                 return false;
3983         }
3984         return radeon_ring_test_lockup(rdev, ring);
3985 }
3986
3987 /*
3988  * RLC
3989  */
3990 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
3991 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
3992
3993 void sumo_rlc_fini(struct radeon_device *rdev)
3994 {
3995         int r;
3996
3997         /* save restore block */
3998         if (rdev->rlc.save_restore_obj) {
3999                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4000                 if (unlikely(r != 0))
4001                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4002                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4003                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4004
4005                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4006                 rdev->rlc.save_restore_obj = NULL;
4007         }
4008
4009         /* clear state block */
4010         if (rdev->rlc.clear_state_obj) {
4011                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4012                 if (unlikely(r != 0))
4013                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4014                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4015                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4016
4017                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4018                 rdev->rlc.clear_state_obj = NULL;
4019         }
4020
4021         /* clear state block */
4022         if (rdev->rlc.cp_table_obj) {
4023                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4024                 if (unlikely(r != 0))
4025                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4026                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4027                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4028
4029                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4030                 rdev->rlc.cp_table_obj = NULL;
4031         }
4032 }
4033
4034 #define CP_ME_TABLE_SIZE    96
4035
4036 int sumo_rlc_init(struct radeon_device *rdev)
4037 {
4038         const u32 *src_ptr;
4039         volatile u32 *dst_ptr;
4040         u32 dws, data, i, j, k, reg_num;
4041         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4042         u64 reg_list_mc_addr;
4043         const struct cs_section_def *cs_data;
4044         int r;
4045
4046         src_ptr = rdev->rlc.reg_list;
4047         dws = rdev->rlc.reg_list_size;
4048         if (rdev->family >= CHIP_BONAIRE) {
4049                 dws += (5 * 16) + 48 + 48 + 64;
4050         }
4051         cs_data = rdev->rlc.cs_data;
4052
4053         if (src_ptr) {
4054                 /* save restore block */
4055                 if (rdev->rlc.save_restore_obj == NULL) {
4056                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4057                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4058                                              NULL, &rdev->rlc.save_restore_obj);
4059                         if (r) {
4060                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4061                                 return r;
4062                         }
4063                 }
4064
4065                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4066                 if (unlikely(r != 0)) {
4067                         sumo_rlc_fini(rdev);
4068                         return r;
4069                 }
4070                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4071                                   &rdev->rlc.save_restore_gpu_addr);
4072                 if (r) {
4073                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4074                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4075                         sumo_rlc_fini(rdev);
4076                         return r;
4077                 }
4078
4079                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4080                 if (r) {
4081                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4082                         sumo_rlc_fini(rdev);
4083                         return r;
4084                 }
4085                 /* write the sr buffer */
4086                 dst_ptr = rdev->rlc.sr_ptr;
4087                 if (rdev->family >= CHIP_TAHITI) {
4088                         /* SI */
4089                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4090                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4091                 } else {
4092                         /* ON/LN/TN */
4093                         /* format:
4094                          * dw0: (reg2 << 16) | reg1
4095                          * dw1: reg1 save space
4096                          * dw2: reg2 save space
4097                          */
4098                         for (i = 0; i < dws; i++) {
4099                                 data = src_ptr[i] >> 2;
4100                                 i++;
4101                                 if (i < dws)
4102                                         data |= (src_ptr[i] >> 2) << 16;
4103                                 j = (((i - 1) * 3) / 2);
4104                                 dst_ptr[j] = cpu_to_le32(data);
4105                         }
4106                         j = ((i * 3) / 2);
4107                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4108                 }
4109                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4110                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4111         }
4112
4113         if (cs_data) {
4114                 /* clear state block */
4115                 if (rdev->family >= CHIP_BONAIRE) {
4116                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4117                 } else if (rdev->family >= CHIP_TAHITI) {
4118                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4119                         dws = rdev->rlc.clear_state_size + (256 / 4);
4120                 } else {
4121                         reg_list_num = 0;
4122                         dws = 0;
4123                         for (i = 0; cs_data[i].section != NULL; i++) {
4124                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4125                                         reg_list_num++;
4126                                         dws += cs_data[i].section[j].reg_count;
4127                                 }
4128                         }
4129                         reg_list_blk_index = (3 * reg_list_num + 2);
4130                         dws += reg_list_blk_index;
4131                         rdev->rlc.clear_state_size = dws;
4132                 }
4133
4134                 if (rdev->rlc.clear_state_obj == NULL) {
4135                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4136                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4137                                              NULL, &rdev->rlc.clear_state_obj);
4138                         if (r) {
4139                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4140                                 sumo_rlc_fini(rdev);
4141                                 return r;
4142                         }
4143                 }
4144                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4145                 if (unlikely(r != 0)) {
4146                         sumo_rlc_fini(rdev);
4147                         return r;
4148                 }
4149                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4150                                   &rdev->rlc.clear_state_gpu_addr);
4151                 if (r) {
4152                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4153                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4154                         sumo_rlc_fini(rdev);
4155                         return r;
4156                 }
4157
4158                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4159                 if (r) {
4160                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4161                         sumo_rlc_fini(rdev);
4162                         return r;
4163                 }
4164                 /* set up the cs buffer */
4165                 dst_ptr = rdev->rlc.cs_ptr;
4166                 if (rdev->family >= CHIP_BONAIRE) {
4167                         cik_get_csb_buffer(rdev, dst_ptr);
4168                 } else if (rdev->family >= CHIP_TAHITI) {
4169                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4170                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4171                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4172                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4173                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4174                 } else {
4175                         reg_list_hdr_blk_index = 0;
4176                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4177                         data = upper_32_bits(reg_list_mc_addr);
4178                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4179                         reg_list_hdr_blk_index++;
4180                         for (i = 0; cs_data[i].section != NULL; i++) {
4181                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4182                                         reg_num = cs_data[i].section[j].reg_count;
4183                                         data = reg_list_mc_addr & 0xffffffff;
4184                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4185                                         reg_list_hdr_blk_index++;
4186
4187                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4188                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4189                                         reg_list_hdr_blk_index++;
4190
4191                                         data = 0x08000000 | (reg_num * 4);
4192                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4193                                         reg_list_hdr_blk_index++;
4194
4195                                         for (k = 0; k < reg_num; k++) {
4196                                                 data = cs_data[i].section[j].extent[k];
4197                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4198                                         }
4199                                         reg_list_mc_addr += reg_num * 4;
4200                                         reg_list_blk_index += reg_num;
4201                                 }
4202                         }
4203                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4204                 }
4205                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4206                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4207         }
4208
4209         if (rdev->rlc.cp_table_size) {
4210                 if (rdev->rlc.cp_table_obj == NULL) {
4211                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4212                                              PAGE_SIZE, true,
4213                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4214                                              NULL, &rdev->rlc.cp_table_obj);
4215                         if (r) {
4216                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4217                                 sumo_rlc_fini(rdev);
4218                                 return r;
4219                         }
4220                 }
4221
4222                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4223                 if (unlikely(r != 0)) {
4224                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4225                         sumo_rlc_fini(rdev);
4226                         return r;
4227                 }
4228                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4229                                   &rdev->rlc.cp_table_gpu_addr);
4230                 if (r) {
4231                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4232                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4233                         sumo_rlc_fini(rdev);
4234                         return r;
4235                 }
4236                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4237                 if (r) {
4238                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4239                         sumo_rlc_fini(rdev);
4240                         return r;
4241                 }
4242
4243                 cik_init_cp_pg_table(rdev);
4244
4245                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4246                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4247
4248         }
4249
4250         return 0;
4251 }
4252
4253 static void evergreen_rlc_start(struct radeon_device *rdev)
4254 {
4255         u32 mask = RLC_ENABLE;
4256
4257         if (rdev->flags & RADEON_IS_IGP) {
4258                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4259         }
4260
4261         WREG32(RLC_CNTL, mask);
4262 }
4263
4264 int evergreen_rlc_resume(struct radeon_device *rdev)
4265 {
4266         u32 i;
4267         const __be32 *fw_data;
4268
4269         if (!rdev->rlc_fw)
4270                 return -EINVAL;
4271
4272         r600_rlc_stop(rdev);
4273
4274         WREG32(RLC_HB_CNTL, 0);
4275
4276         if (rdev->flags & RADEON_IS_IGP) {
4277                 if (rdev->family == CHIP_ARUBA) {
4278                         u32 always_on_bitmap =
4279                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4280                         /* find out the number of active simds */
4281                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4282                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4283                         tmp = hweight32(~tmp);
4284                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4285                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4286                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4287                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4288                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4289                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4290                         }
4291                 } else {
4292                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4293                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4294                 }
4295                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4296                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4297         } else {
4298                 WREG32(RLC_HB_BASE, 0);
4299                 WREG32(RLC_HB_RPTR, 0);
4300                 WREG32(RLC_HB_WPTR, 0);
4301                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4302                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4303         }
4304         WREG32(RLC_MC_CNTL, 0);
4305         WREG32(RLC_UCODE_CNTL, 0);
4306
4307         fw_data = (const __be32 *)rdev->rlc_fw->data;
4308         if (rdev->family >= CHIP_ARUBA) {
4309                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4310                         WREG32(RLC_UCODE_ADDR, i);
4311                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4312                 }
4313         } else if (rdev->family >= CHIP_CAYMAN) {
4314                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4315                         WREG32(RLC_UCODE_ADDR, i);
4316                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4317                 }
4318         } else {
4319                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4320                         WREG32(RLC_UCODE_ADDR, i);
4321                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4322                 }
4323         }
4324         WREG32(RLC_UCODE_ADDR, 0);
4325
4326         evergreen_rlc_start(rdev);
4327
4328         return 0;
4329 }
4330
4331 /* Interrupts */
4332
4333 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4334 {
4335         if (crtc >= rdev->num_crtc)
4336                 return 0;
4337         else
4338                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4339 }
4340
4341 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4342 {
4343         u32 tmp;
4344
4345         if (rdev->family >= CHIP_CAYMAN) {
4346                 cayman_cp_int_cntl_setup(rdev, 0,
4347                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4348                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4349                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4350                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4351                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4352         } else
4353                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4354         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4355         WREG32(DMA_CNTL, tmp);
4356         WREG32(GRBM_INT_CNTL, 0);
4357         WREG32(SRBM_INT_CNTL, 0);
4358         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4359         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4360         if (rdev->num_crtc >= 4) {
4361                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4362                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4363         }
4364         if (rdev->num_crtc >= 6) {
4365                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4366                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4367         }
4368
4369         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4370         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4371         if (rdev->num_crtc >= 4) {
4372                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4373                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4374         }
4375         if (rdev->num_crtc >= 6) {
4376                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4377                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4378         }
4379
4380         /* only one DAC on DCE5 */
4381         if (!ASIC_IS_DCE5(rdev))
4382                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4383         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4384
4385         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4386         WREG32(DC_HPD1_INT_CONTROL, tmp);
4387         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4388         WREG32(DC_HPD2_INT_CONTROL, tmp);
4389         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4390         WREG32(DC_HPD3_INT_CONTROL, tmp);
4391         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4392         WREG32(DC_HPD4_INT_CONTROL, tmp);
4393         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4394         WREG32(DC_HPD5_INT_CONTROL, tmp);
4395         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4396         WREG32(DC_HPD6_INT_CONTROL, tmp);
4397
4398 }
4399
4400 int evergreen_irq_set(struct radeon_device *rdev)
4401 {
4402         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4403         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4404         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4405         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4406         u32 grbm_int_cntl = 0;
4407         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4408         u32 dma_cntl, dma_cntl1 = 0;
4409         u32 thermal_int = 0;
4410
4411         if (!rdev->irq.installed) {
4412                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4413                 return -EINVAL;
4414         }
4415         /* don't enable anything if the ih is disabled */
4416         if (!rdev->ih.enabled) {
4417                 r600_disable_interrupts(rdev);
4418                 /* force the active interrupt state to all disabled */
4419                 evergreen_disable_interrupt_state(rdev);
4420                 return 0;
4421         }
4422
4423         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4424         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4425         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4426         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4427         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4428         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4429         if (rdev->family == CHIP_ARUBA)
4430                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4431                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4432         else
4433                 thermal_int = RREG32(CG_THERMAL_INT) &
4434                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4435
4436         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4437         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4438         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4439         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4440         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4441         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4442
4443         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4444
4445         if (rdev->family >= CHIP_CAYMAN) {
4446                 /* enable CP interrupts on all rings */
4447                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4448                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4449                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4450                 }
4451                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4452                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4453                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4454                 }
4455                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4456                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4457                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4458                 }
4459         } else {
4460                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4461                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4462                         cp_int_cntl |= RB_INT_ENABLE;
4463                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4464                 }
4465         }
4466
4467         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4468                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4469                 dma_cntl |= TRAP_ENABLE;
4470         }
4471
4472         if (rdev->family >= CHIP_CAYMAN) {
4473                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4474                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4475                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4476                         dma_cntl1 |= TRAP_ENABLE;
4477                 }
4478         }
4479
4480         if (rdev->irq.dpm_thermal) {
4481                 DRM_DEBUG("dpm thermal\n");
4482                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4483         }
4484
4485         if (rdev->irq.crtc_vblank_int[0] ||
4486             atomic_read(&rdev->irq.pflip[0])) {
4487                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4488                 crtc1 |= VBLANK_INT_MASK;
4489         }
4490         if (rdev->irq.crtc_vblank_int[1] ||
4491             atomic_read(&rdev->irq.pflip[1])) {
4492                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4493                 crtc2 |= VBLANK_INT_MASK;
4494         }
4495         if (rdev->irq.crtc_vblank_int[2] ||
4496             atomic_read(&rdev->irq.pflip[2])) {
4497                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4498                 crtc3 |= VBLANK_INT_MASK;
4499         }
4500         if (rdev->irq.crtc_vblank_int[3] ||
4501             atomic_read(&rdev->irq.pflip[3])) {
4502                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4503                 crtc4 |= VBLANK_INT_MASK;
4504         }
4505         if (rdev->irq.crtc_vblank_int[4] ||
4506             atomic_read(&rdev->irq.pflip[4])) {
4507                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4508                 crtc5 |= VBLANK_INT_MASK;
4509         }
4510         if (rdev->irq.crtc_vblank_int[5] ||
4511             atomic_read(&rdev->irq.pflip[5])) {
4512                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4513                 crtc6 |= VBLANK_INT_MASK;
4514         }
4515         if (rdev->irq.hpd[0]) {
4516                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4517                 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4518         }
4519         if (rdev->irq.hpd[1]) {
4520                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4521                 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4522         }
4523         if (rdev->irq.hpd[2]) {
4524                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4525                 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4526         }
4527         if (rdev->irq.hpd[3]) {
4528                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4529                 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4530         }
4531         if (rdev->irq.hpd[4]) {
4532                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4533                 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4534         }
4535         if (rdev->irq.hpd[5]) {
4536                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4537                 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4538         }
4539         if (rdev->irq.afmt[0]) {
4540                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4541                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4542         }
4543         if (rdev->irq.afmt[1]) {
4544                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4545                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4546         }
4547         if (rdev->irq.afmt[2]) {
4548                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4549                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4550         }
4551         if (rdev->irq.afmt[3]) {
4552                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4553                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4554         }
4555         if (rdev->irq.afmt[4]) {
4556                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4557                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4558         }
4559         if (rdev->irq.afmt[5]) {
4560                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4561                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4562         }
4563
4564         if (rdev->family >= CHIP_CAYMAN) {
4565                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4566                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4567                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4568         } else
4569                 WREG32(CP_INT_CNTL, cp_int_cntl);
4570
4571         WREG32(DMA_CNTL, dma_cntl);
4572
4573         if (rdev->family >= CHIP_CAYMAN)
4574                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4575
4576         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4577
4578         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4579         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4580         if (rdev->num_crtc >= 4) {
4581                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4582                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4583         }
4584         if (rdev->num_crtc >= 6) {
4585                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4586                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4587         }
4588
4589         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4590                GRPH_PFLIP_INT_MASK);
4591         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4592                GRPH_PFLIP_INT_MASK);
4593         if (rdev->num_crtc >= 4) {
4594                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4595                        GRPH_PFLIP_INT_MASK);
4596                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4597                        GRPH_PFLIP_INT_MASK);
4598         }
4599         if (rdev->num_crtc >= 6) {
4600                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4601                        GRPH_PFLIP_INT_MASK);
4602                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4603                        GRPH_PFLIP_INT_MASK);
4604         }
4605
4606         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4607         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4608         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4609         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4610         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4611         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4612         if (rdev->family == CHIP_ARUBA)
4613                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4614         else
4615                 WREG32(CG_THERMAL_INT, thermal_int);
4616
4617         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4618         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4619         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4620         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4621         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4622         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4623
4624         /* posting read */
4625         RREG32(SRBM_STATUS);
4626
4627         return 0;
4628 }
4629
4630 static void evergreen_irq_ack(struct radeon_device *rdev)
4631 {
4632         u32 tmp;
4633
4634         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4635         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4636         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4637         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4638         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4639         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4640         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4641         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4642         if (rdev->num_crtc >= 4) {
4643                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4644                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4645         }
4646         if (rdev->num_crtc >= 6) {
4647                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4648                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4649         }
4650
4651         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4652         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4653         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4654         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4655         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4656         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4657
4658         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4659                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4660         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4661                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4662         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4663                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4664         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4665                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4666         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4667                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4668         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4669                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4670
4671         if (rdev->num_crtc >= 4) {
4672                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4673                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4674                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4675                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4676                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4677                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4678                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4679                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4680                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4681                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4682                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4683                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4684         }
4685
4686         if (rdev->num_crtc >= 6) {
4687                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4688                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4689                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4690                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4691                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4692                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4693                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4694                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4695                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4696                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4697                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4698                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4699         }
4700
4701         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4702                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4703                 tmp |= DC_HPDx_INT_ACK;
4704                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4705         }
4706         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4707                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4708                 tmp |= DC_HPDx_INT_ACK;
4709                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4710         }
4711         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4712                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4713                 tmp |= DC_HPDx_INT_ACK;
4714                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4715         }
4716         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4717                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4718                 tmp |= DC_HPDx_INT_ACK;
4719                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4720         }
4721         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4722                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4723                 tmp |= DC_HPDx_INT_ACK;
4724                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4725         }
4726         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4727                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4728                 tmp |= DC_HPDx_INT_ACK;
4729                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4730         }
4731
4732         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4733                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4734                 tmp |= DC_HPDx_RX_INT_ACK;
4735                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4736         }
4737         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4738                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4739                 tmp |= DC_HPDx_RX_INT_ACK;
4740                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4741         }
4742         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4743                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4744                 tmp |= DC_HPDx_RX_INT_ACK;
4745                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4746         }
4747         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4748                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4749                 tmp |= DC_HPDx_RX_INT_ACK;
4750                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4751         }
4752         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4753                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4754                 tmp |= DC_HPDx_RX_INT_ACK;
4755                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4756         }
4757         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4758                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4759                 tmp |= DC_HPDx_RX_INT_ACK;
4760                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4761         }
4762
4763         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4764                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4765                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4766                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4767         }
4768         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4769                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4770                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4771                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4772         }
4773         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4774                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4775                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4776                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4777         }
4778         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4779                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4780                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4781                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4782         }
4783         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4784                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4785                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4786                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4787         }
4788         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4789                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4790                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4791                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4792         }
4793 }
4794
4795 static void evergreen_irq_disable(struct radeon_device *rdev)
4796 {
4797         r600_disable_interrupts(rdev);
4798         /* Wait and acknowledge irq */
4799         mdelay(1);
4800         evergreen_irq_ack(rdev);
4801         evergreen_disable_interrupt_state(rdev);
4802 }
4803
4804 void evergreen_irq_suspend(struct radeon_device *rdev)
4805 {
4806         evergreen_irq_disable(rdev);
4807         r600_rlc_stop(rdev);
4808 }
4809
4810 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4811 {
4812         u32 wptr, tmp;
4813
4814         if (rdev->wb.enabled)
4815                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4816         else
4817                 wptr = RREG32(IH_RB_WPTR);
4818
4819         if (wptr & RB_OVERFLOW) {
4820                 wptr &= ~RB_OVERFLOW;
4821                 /* When a ring buffer overflow happen start parsing interrupt
4822                  * from the last not overwritten vector (wptr + 16). Hopefully
4823                  * this should allow us to catchup.
4824                  */
4825                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4826                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4827                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4828                 tmp = RREG32(IH_RB_CNTL);
4829                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4830                 WREG32(IH_RB_CNTL, tmp);
4831         }
4832         return (wptr & rdev->ih.ptr_mask);
4833 }
4834
4835 int evergreen_irq_process(struct radeon_device *rdev)
4836 {
4837         u32 wptr;
4838         u32 rptr;
4839         u32 src_id, src_data;
4840         u32 ring_index;
4841         bool queue_hotplug = false;
4842         bool queue_hdmi = false;
4843         bool queue_dp = false;
4844         bool queue_thermal = false;
4845         u32 status, addr;
4846
4847         if (!rdev->ih.enabled || rdev->shutdown)
4848                 return IRQ_NONE;
4849
4850         wptr = evergreen_get_ih_wptr(rdev);
4851
4852 restart_ih:
4853         /* is somebody else already processing irqs? */
4854         if (atomic_xchg(&rdev->ih.lock, 1))
4855                 return IRQ_NONE;
4856
4857         rptr = rdev->ih.rptr;
4858         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4859
4860         /* Order reading of wptr vs. reading of IH ring data */
4861         rmb();
4862
4863         /* display interrupts */
4864         evergreen_irq_ack(rdev);
4865
4866         while (rptr != wptr) {
4867                 /* wptr/rptr are in bytes! */
4868                 ring_index = rptr / 4;
4869                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4870                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4871
4872                 switch (src_id) {
4873                 case 1: /* D1 vblank/vline */
4874                         switch (src_data) {
4875                         case 0: /* D1 vblank */
4876                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4877                                         if (rdev->irq.crtc_vblank_int[0]) {
4878                                                 drm_handle_vblank(rdev->ddev, 0);
4879                                                 rdev->pm.vblank_sync = true;
4880                                                 wake_up(&rdev->irq.vblank_queue);
4881                                         }
4882                                         if (atomic_read(&rdev->irq.pflip[0]))
4883                                                 radeon_crtc_handle_vblank(rdev, 0);
4884                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4885                                         DRM_DEBUG("IH: D1 vblank\n");
4886                                 }
4887                                 break;
4888                         case 1: /* D1 vline */
4889                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4890                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4891                                         DRM_DEBUG("IH: D1 vline\n");
4892                                 }
4893                                 break;
4894                         default:
4895                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4896                                 break;
4897                         }
4898                         break;
4899                 case 2: /* D2 vblank/vline */
4900                         switch (src_data) {
4901                         case 0: /* D2 vblank */
4902                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4903                                         if (rdev->irq.crtc_vblank_int[1]) {
4904                                                 drm_handle_vblank(rdev->ddev, 1);
4905                                                 rdev->pm.vblank_sync = true;
4906                                                 wake_up(&rdev->irq.vblank_queue);
4907                                         }
4908                                         if (atomic_read(&rdev->irq.pflip[1]))
4909                                                 radeon_crtc_handle_vblank(rdev, 1);
4910                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4911                                         DRM_DEBUG("IH: D2 vblank\n");
4912                                 }
4913                                 break;
4914                         case 1: /* D2 vline */
4915                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4916                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4917                                         DRM_DEBUG("IH: D2 vline\n");
4918                                 }
4919                                 break;
4920                         default:
4921                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4922                                 break;
4923                         }
4924                         break;
4925                 case 3: /* D3 vblank/vline */
4926                         switch (src_data) {
4927                         case 0: /* D3 vblank */
4928                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4929                                         if (rdev->irq.crtc_vblank_int[2]) {
4930                                                 drm_handle_vblank(rdev->ddev, 2);
4931                                                 rdev->pm.vblank_sync = true;
4932                                                 wake_up(&rdev->irq.vblank_queue);
4933                                         }
4934                                         if (atomic_read(&rdev->irq.pflip[2]))
4935                                                 radeon_crtc_handle_vblank(rdev, 2);
4936                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4937                                         DRM_DEBUG("IH: D3 vblank\n");
4938                                 }
4939                                 break;
4940                         case 1: /* D3 vline */
4941                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4942                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4943                                         DRM_DEBUG("IH: D3 vline\n");
4944                                 }
4945                                 break;
4946                         default:
4947                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4948                                 break;
4949                         }
4950                         break;
4951                 case 4: /* D4 vblank/vline */
4952                         switch (src_data) {
4953                         case 0: /* D4 vblank */
4954                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4955                                         if (rdev->irq.crtc_vblank_int[3]) {
4956                                                 drm_handle_vblank(rdev->ddev, 3);
4957                                                 rdev->pm.vblank_sync = true;
4958                                                 wake_up(&rdev->irq.vblank_queue);
4959                                         }
4960                                         if (atomic_read(&rdev->irq.pflip[3]))
4961                                                 radeon_crtc_handle_vblank(rdev, 3);
4962                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4963                                         DRM_DEBUG("IH: D4 vblank\n");
4964                                 }
4965                                 break;
4966                         case 1: /* D4 vline */
4967                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4968                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4969                                         DRM_DEBUG("IH: D4 vline\n");
4970                                 }
4971                                 break;
4972                         default:
4973                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4974                                 break;
4975                         }
4976                         break;
4977                 case 5: /* D5 vblank/vline */
4978                         switch (src_data) {
4979                         case 0: /* D5 vblank */
4980                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4981                                         if (rdev->irq.crtc_vblank_int[4]) {
4982                                                 drm_handle_vblank(rdev->ddev, 4);
4983                                                 rdev->pm.vblank_sync = true;
4984                                                 wake_up(&rdev->irq.vblank_queue);
4985                                         }
4986                                         if (atomic_read(&rdev->irq.pflip[4]))
4987                                                 radeon_crtc_handle_vblank(rdev, 4);
4988                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4989                                         DRM_DEBUG("IH: D5 vblank\n");
4990                                 }
4991                                 break;
4992                         case 1: /* D5 vline */
4993                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4994                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4995                                         DRM_DEBUG("IH: D5 vline\n");
4996                                 }
4997                                 break;
4998                         default:
4999                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5000                                 break;
5001                         }
5002                         break;
5003                 case 6: /* D6 vblank/vline */
5004                         switch (src_data) {
5005                         case 0: /* D6 vblank */
5006                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5007                                         if (rdev->irq.crtc_vblank_int[5]) {
5008                                                 drm_handle_vblank(rdev->ddev, 5);
5009                                                 rdev->pm.vblank_sync = true;
5010                                                 wake_up(&rdev->irq.vblank_queue);
5011                                         }
5012                                         if (atomic_read(&rdev->irq.pflip[5]))
5013                                                 radeon_crtc_handle_vblank(rdev, 5);
5014                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5015                                         DRM_DEBUG("IH: D6 vblank\n");
5016                                 }
5017                                 break;
5018                         case 1: /* D6 vline */
5019                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5020                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5021                                         DRM_DEBUG("IH: D6 vline\n");
5022                                 }
5023                                 break;
5024                         default:
5025                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5026                                 break;
5027                         }
5028                         break;
5029                 case 8: /* D1 page flip */
5030                 case 10: /* D2 page flip */
5031                 case 12: /* D3 page flip */
5032                 case 14: /* D4 page flip */
5033                 case 16: /* D5 page flip */
5034                 case 18: /* D6 page flip */
5035                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5036                         if (radeon_use_pflipirq > 0)
5037                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5038                         break;
5039                 case 42: /* HPD hotplug */
5040                         switch (src_data) {
5041                         case 0:
5042                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5043                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5044                                         queue_hotplug = true;
5045                                         DRM_DEBUG("IH: HPD1\n");
5046                                 }
5047                                 break;
5048                         case 1:
5049                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5050                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5051                                         queue_hotplug = true;
5052                                         DRM_DEBUG("IH: HPD2\n");
5053                                 }
5054                                 break;
5055                         case 2:
5056                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5057                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5058                                         queue_hotplug = true;
5059                                         DRM_DEBUG("IH: HPD3\n");
5060                                 }
5061                                 break;
5062                         case 3:
5063                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5064                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5065                                         queue_hotplug = true;
5066                                         DRM_DEBUG("IH: HPD4\n");
5067                                 }
5068                                 break;
5069                         case 4:
5070                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5071                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5072                                         queue_hotplug = true;
5073                                         DRM_DEBUG("IH: HPD5\n");
5074                                 }
5075                                 break;
5076                         case 5:
5077                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5078                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5079                                         queue_hotplug = true;
5080                                         DRM_DEBUG("IH: HPD6\n");
5081                                 }
5082                                 break;
5083                         case 6:
5084                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
5085                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5086                                         queue_dp = true;
5087                                         DRM_DEBUG("IH: HPD_RX 1\n");
5088                                 }
5089                                 break;
5090                         case 7:
5091                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
5092                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5093                                         queue_dp = true;
5094                                         DRM_DEBUG("IH: HPD_RX 2\n");
5095                                 }
5096                                 break;
5097                         case 8:
5098                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
5099                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5100                                         queue_dp = true;
5101                                         DRM_DEBUG("IH: HPD_RX 3\n");
5102                                 }
5103                                 break;
5104                         case 9:
5105                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
5106                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5107                                         queue_dp = true;
5108                                         DRM_DEBUG("IH: HPD_RX 4\n");
5109                                 }
5110                                 break;
5111                         case 10:
5112                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
5113                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5114                                         queue_dp = true;
5115                                         DRM_DEBUG("IH: HPD_RX 5\n");
5116                                 }
5117                                 break;
5118                         case 11:
5119                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
5120                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5121                                         queue_dp = true;
5122                                         DRM_DEBUG("IH: HPD_RX 6\n");
5123                                 }
5124                                 break;
5125                         default:
5126                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5127                                 break;
5128                         }
5129                         break;
5130                 case 44: /* hdmi */
5131                         switch (src_data) {
5132                         case 0:
5133                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5134                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5135                                         queue_hdmi = true;
5136                                         DRM_DEBUG("IH: HDMI0\n");
5137                                 }
5138                                 break;
5139                         case 1:
5140                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5141                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5142                                         queue_hdmi = true;
5143                                         DRM_DEBUG("IH: HDMI1\n");
5144                                 }
5145                                 break;
5146                         case 2:
5147                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5148                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5149                                         queue_hdmi = true;
5150                                         DRM_DEBUG("IH: HDMI2\n");
5151                                 }
5152                                 break;
5153                         case 3:
5154                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5155                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5156                                         queue_hdmi = true;
5157                                         DRM_DEBUG("IH: HDMI3\n");
5158                                 }
5159                                 break;
5160                         case 4:
5161                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5162                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5163                                         queue_hdmi = true;
5164                                         DRM_DEBUG("IH: HDMI4\n");
5165                                 }
5166                                 break;
5167                         case 5:
5168                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5169                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5170                                         queue_hdmi = true;
5171                                         DRM_DEBUG("IH: HDMI5\n");
5172                                 }
5173                                 break;
5174                         default:
5175                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5176                                 break;
5177                         }
5178                 case 96:
5179                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5180                         WREG32(SRBM_INT_ACK, 0x1);
5181                         break;
5182                 case 124: /* UVD */
5183                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5184                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5185                         break;
5186                 case 146:
5187                 case 147:
5188                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5189                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5190                         /* reset addr and status */
5191                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5192                         if (addr == 0x0 && status == 0x0)
5193                                 break;
5194                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5195                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5196                                 addr);
5197                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5198                                 status);
5199                         cayman_vm_decode_fault(rdev, status, addr);
5200                         break;
5201                 case 176: /* CP_INT in ring buffer */
5202                 case 177: /* CP_INT in IB1 */
5203                 case 178: /* CP_INT in IB2 */
5204                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5205                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5206                         break;
5207                 case 181: /* CP EOP event */
5208                         DRM_DEBUG("IH: CP EOP\n");
5209                         if (rdev->family >= CHIP_CAYMAN) {
5210                                 switch (src_data) {
5211                                 case 0:
5212                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5213                                         break;
5214                                 case 1:
5215                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5216                                         break;
5217                                 case 2:
5218                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5219                                         break;
5220                                 }
5221                         } else
5222                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5223                         break;
5224                 case 224: /* DMA trap event */
5225                         DRM_DEBUG("IH: DMA trap\n");
5226                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5227                         break;
5228                 case 230: /* thermal low to high */
5229                         DRM_DEBUG("IH: thermal low to high\n");
5230                         rdev->pm.dpm.thermal.high_to_low = false;
5231                         queue_thermal = true;
5232                         break;
5233                 case 231: /* thermal high to low */
5234                         DRM_DEBUG("IH: thermal high to low\n");
5235                         rdev->pm.dpm.thermal.high_to_low = true;
5236                         queue_thermal = true;
5237                         break;
5238                 case 233: /* GUI IDLE */
5239                         DRM_DEBUG("IH: GUI idle\n");
5240                         break;
5241                 case 244: /* DMA trap event */
5242                         if (rdev->family >= CHIP_CAYMAN) {
5243                                 DRM_DEBUG("IH: DMA1 trap\n");
5244                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5245                         }
5246                         break;
5247                 default:
5248                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5249                         break;
5250                 }
5251
5252                 /* wptr/rptr are in bytes! */
5253                 rptr += 16;
5254                 rptr &= rdev->ih.ptr_mask;
5255                 WREG32(IH_RB_RPTR, rptr);
5256         }
5257         if (queue_dp)
5258                 schedule_work(&rdev->dp_work);
5259         if (queue_hotplug)
5260                 schedule_work(&rdev->hotplug_work);
5261         if (queue_hdmi)
5262                 schedule_work(&rdev->audio_work);
5263         if (queue_thermal && rdev->pm.dpm_enabled)
5264                 schedule_work(&rdev->pm.dpm.thermal.work);
5265         rdev->ih.rptr = rptr;
5266         atomic_set(&rdev->ih.lock, 0);
5267
5268         /* make sure wptr hasn't changed while processing */
5269         wptr = evergreen_get_ih_wptr(rdev);
5270         if (wptr != rptr)
5271                 goto restart_ih;
5272
5273         return IRQ_HANDLED;
5274 }
5275
5276 static int evergreen_startup(struct radeon_device *rdev)
5277 {
5278         struct radeon_ring *ring;
5279         int r;
5280
5281         /* enable pcie gen2 link */
5282         evergreen_pcie_gen2_enable(rdev);
5283         /* enable aspm */
5284         evergreen_program_aspm(rdev);
5285
5286         /* scratch needs to be initialized before MC */
5287         r = r600_vram_scratch_init(rdev);
5288         if (r)
5289                 return r;
5290
5291         evergreen_mc_program(rdev);
5292
5293         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5294                 r = ni_mc_load_microcode(rdev);
5295                 if (r) {
5296                         DRM_ERROR("Failed to load MC firmware!\n");
5297                         return r;
5298                 }
5299         }
5300
5301         if (rdev->flags & RADEON_IS_AGP) {
5302                 evergreen_agp_enable(rdev);
5303         } else {
5304                 r = evergreen_pcie_gart_enable(rdev);
5305                 if (r)
5306                         return r;
5307         }
5308         evergreen_gpu_init(rdev);
5309
5310         /* allocate rlc buffers */
5311         if (rdev->flags & RADEON_IS_IGP) {
5312                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5313                 rdev->rlc.reg_list_size =
5314                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5315                 rdev->rlc.cs_data = evergreen_cs_data;
5316                 r = sumo_rlc_init(rdev);
5317                 if (r) {
5318                         DRM_ERROR("Failed to init rlc BOs!\n");
5319                         return r;
5320                 }
5321         }
5322
5323         /* allocate wb buffer */
5324         r = radeon_wb_init(rdev);
5325         if (r)
5326                 return r;
5327
5328         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5329         if (r) {
5330                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5331                 return r;
5332         }
5333
5334         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5335         if (r) {
5336                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5337                 return r;
5338         }
5339
5340         r = uvd_v2_2_resume(rdev);
5341         if (!r) {
5342                 r = radeon_fence_driver_start_ring(rdev,
5343                                                    R600_RING_TYPE_UVD_INDEX);
5344                 if (r)
5345                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5346         }
5347
5348         if (r)
5349                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5350
5351         /* Enable IRQ */
5352         if (!rdev->irq.installed) {
5353                 r = radeon_irq_kms_init(rdev);
5354                 if (r)
5355                         return r;
5356         }
5357
5358         r = r600_irq_init(rdev);
5359         if (r) {
5360                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5361                 radeon_irq_kms_fini(rdev);
5362                 return r;
5363         }
5364         evergreen_irq_set(rdev);
5365
5366         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5367         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5368                              RADEON_CP_PACKET2);
5369         if (r)
5370                 return r;
5371
5372         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5373         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5374                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5375         if (r)
5376                 return r;
5377
5378         r = evergreen_cp_load_microcode(rdev);
5379         if (r)
5380                 return r;
5381         r = evergreen_cp_resume(rdev);
5382         if (r)
5383                 return r;
5384         r = r600_dma_resume(rdev);
5385         if (r)
5386                 return r;
5387
5388         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5389         if (ring->ring_size) {
5390                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5391                                      RADEON_CP_PACKET2);
5392                 if (!r)
5393                         r = uvd_v1_0_init(rdev);
5394
5395                 if (r)
5396                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5397         }
5398
5399         r = radeon_ib_pool_init(rdev);
5400         if (r) {
5401                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5402                 return r;
5403         }
5404
5405         r = radeon_audio_init(rdev);
5406         if (r) {
5407                 DRM_ERROR("radeon: audio init failed\n");
5408                 return r;
5409         }
5410
5411         return 0;
5412 }
5413
5414 int evergreen_resume(struct radeon_device *rdev)
5415 {
5416         int r;
5417
5418         /* reset the asic, the gfx blocks are often in a bad state
5419          * after the driver is unloaded or after a resume
5420          */
5421         if (radeon_asic_reset(rdev))
5422                 dev_warn(rdev->dev, "GPU reset failed !\n");
5423         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5424          * posting will perform necessary task to bring back GPU into good
5425          * shape.
5426          */
5427         /* post card */
5428         atom_asic_init(rdev->mode_info.atom_context);
5429
5430         /* init golden registers */
5431         evergreen_init_golden_registers(rdev);
5432
5433         if (rdev->pm.pm_method == PM_METHOD_DPM)
5434                 radeon_pm_resume(rdev);
5435
5436         rdev->accel_working = true;
5437         r = evergreen_startup(rdev);
5438         if (r) {
5439                 DRM_ERROR("evergreen startup failed on resume\n");
5440                 rdev->accel_working = false;
5441                 return r;
5442         }
5443
5444         return r;
5445
5446 }
5447
5448 int evergreen_suspend(struct radeon_device *rdev)
5449 {
5450         radeon_pm_suspend(rdev);
5451         radeon_audio_fini(rdev);
5452         uvd_v1_0_fini(rdev);
5453         radeon_uvd_suspend(rdev);
5454         r700_cp_stop(rdev);
5455         r600_dma_stop(rdev);
5456         evergreen_irq_suspend(rdev);
5457         radeon_wb_disable(rdev);
5458         evergreen_pcie_gart_disable(rdev);
5459
5460         return 0;
5461 }
5462
5463 /* Plan is to move initialization in that function and use
5464  * helper function so that radeon_device_init pretty much
5465  * do nothing more than calling asic specific function. This
5466  * should also allow to remove a bunch of callback function
5467  * like vram_info.
5468  */
5469 int evergreen_init(struct radeon_device *rdev)
5470 {
5471         int r;
5472
5473         /* Read BIOS */
5474         if (!radeon_get_bios(rdev)) {
5475                 if (ASIC_IS_AVIVO(rdev))
5476                         return -EINVAL;
5477         }
5478         /* Must be an ATOMBIOS */
5479         if (!rdev->is_atom_bios) {
5480                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5481                 return -EINVAL;
5482         }
5483         r = radeon_atombios_init(rdev);
5484         if (r)
5485                 return r;
5486         /* reset the asic, the gfx blocks are often in a bad state
5487          * after the driver is unloaded or after a resume
5488          */
5489         if (radeon_asic_reset(rdev))
5490                 dev_warn(rdev->dev, "GPU reset failed !\n");
5491         /* Post card if necessary */
5492         if (!radeon_card_posted(rdev)) {
5493                 if (!rdev->bios) {
5494                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5495                         return -EINVAL;
5496                 }
5497                 DRM_INFO("GPU not posted. posting now...\n");
5498                 atom_asic_init(rdev->mode_info.atom_context);
5499         }
5500         /* init golden registers */
5501         evergreen_init_golden_registers(rdev);
5502         /* Initialize scratch registers */
5503         r600_scratch_init(rdev);
5504         /* Initialize surface registers */
5505         radeon_surface_init(rdev);
5506         /* Initialize clocks */
5507         radeon_get_clock_info(rdev->ddev);
5508         /* Fence driver */
5509         r = radeon_fence_driver_init(rdev);
5510         if (r)
5511                 return r;
5512         /* initialize AGP */
5513         if (rdev->flags & RADEON_IS_AGP) {
5514                 r = radeon_agp_init(rdev);
5515                 if (r)
5516                         radeon_agp_disable(rdev);
5517         }
5518         /* initialize memory controller */
5519         r = evergreen_mc_init(rdev);
5520         if (r)
5521                 return r;
5522         /* Memory manager */
5523         r = radeon_bo_init(rdev);
5524         if (r)
5525                 return r;
5526
5527         if (ASIC_IS_DCE5(rdev)) {
5528                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5529                         r = ni_init_microcode(rdev);
5530                         if (r) {
5531                                 DRM_ERROR("Failed to load firmware!\n");
5532                                 return r;
5533                         }
5534                 }
5535         } else {
5536                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5537                         r = r600_init_microcode(rdev);
5538                         if (r) {
5539                                 DRM_ERROR("Failed to load firmware!\n");
5540                                 return r;
5541                         }
5542                 }
5543         }
5544
5545         /* Initialize power management */
5546         radeon_pm_init(rdev);
5547
5548         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5549         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5550
5551         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5552         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5553
5554         r = radeon_uvd_init(rdev);
5555         if (!r) {
5556                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5557                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5558                                4096);
5559         }
5560
5561         rdev->ih.ring_obj = NULL;
5562         r600_ih_ring_init(rdev, 64 * 1024);
5563
5564         r = r600_pcie_gart_init(rdev);
5565         if (r)
5566                 return r;
5567
5568         rdev->accel_working = true;
5569         r = evergreen_startup(rdev);
5570         if (r) {
5571                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5572                 r700_cp_fini(rdev);
5573                 r600_dma_fini(rdev);
5574                 r600_irq_fini(rdev);
5575                 if (rdev->flags & RADEON_IS_IGP)
5576                         sumo_rlc_fini(rdev);
5577                 radeon_wb_fini(rdev);
5578                 radeon_ib_pool_fini(rdev);
5579                 radeon_irq_kms_fini(rdev);
5580                 evergreen_pcie_gart_fini(rdev);
5581                 rdev->accel_working = false;
5582         }
5583
5584         /* Don't start up if the MC ucode is missing on BTC parts.
5585          * The default clocks and voltages before the MC ucode
5586          * is loaded are not suffient for advanced operations.
5587          */
5588         if (ASIC_IS_DCE5(rdev)) {
5589                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5590                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5591                         return -EINVAL;
5592                 }
5593         }
5594
5595         return 0;
5596 }
5597
5598 void evergreen_fini(struct radeon_device *rdev)
5599 {
5600         radeon_pm_fini(rdev);
5601         radeon_audio_fini(rdev);
5602         r700_cp_fini(rdev);
5603         r600_dma_fini(rdev);
5604         r600_irq_fini(rdev);
5605         if (rdev->flags & RADEON_IS_IGP)
5606                 sumo_rlc_fini(rdev);
5607         radeon_wb_fini(rdev);
5608         radeon_ib_pool_fini(rdev);
5609         radeon_irq_kms_fini(rdev);
5610         uvd_v1_0_fini(rdev);
5611         radeon_uvd_fini(rdev);
5612         evergreen_pcie_gart_fini(rdev);
5613         r600_vram_scratch_fini(rdev);
5614         radeon_gem_fini(rdev);
5615         radeon_fence_driver_fini(rdev);
5616         radeon_agp_fini(rdev);
5617         radeon_bo_fini(rdev);
5618         radeon_atombios_fini(rdev);
5619         kfree(rdev->bios);
5620         rdev->bios = NULL;
5621 }
5622
5623 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5624 {
5625         u32 link_width_cntl, speed_cntl;
5626
5627         if (radeon_pcie_gen2 == 0)
5628                 return;
5629
5630         if (rdev->flags & RADEON_IS_IGP)
5631                 return;
5632
5633         if (!(rdev->flags & RADEON_IS_PCIE))
5634                 return;
5635
5636         /* x2 cards have a special sequence */
5637         if (ASIC_IS_X2(rdev))
5638                 return;
5639
5640         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5641                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5642                 return;
5643
5644         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5645         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5646                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5647                 return;
5648         }
5649
5650         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5651
5652         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5653             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5654
5655                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5656                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5657                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5658
5659                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5660                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5661                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5662
5663                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5664                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5665                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5666
5667                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5668                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5669                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5670
5671                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5672                 speed_cntl |= LC_GEN2_EN_STRAP;
5673                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5674
5675         } else {
5676                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5677                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5678                 if (1)
5679                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5680                 else
5681                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5682                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5683         }
5684 }
5685
5686 void evergreen_program_aspm(struct radeon_device *rdev)
5687 {
5688         u32 data, orig;
5689         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5690         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5691         /* fusion_platform = true
5692          * if the system is a fusion system
5693          * (APU or DGPU in a fusion system).
5694          * todo: check if the system is a fusion platform.
5695          */
5696         bool fusion_platform = false;
5697
5698         if (radeon_aspm == 0)
5699                 return;
5700
5701         if (!(rdev->flags & RADEON_IS_PCIE))
5702                 return;
5703
5704         switch (rdev->family) {
5705         case CHIP_CYPRESS:
5706         case CHIP_HEMLOCK:
5707         case CHIP_JUNIPER:
5708         case CHIP_REDWOOD:
5709         case CHIP_CEDAR:
5710         case CHIP_SUMO:
5711         case CHIP_SUMO2:
5712         case CHIP_PALM:
5713         case CHIP_ARUBA:
5714                 disable_l0s = true;
5715                 break;
5716         default:
5717                 disable_l0s = false;
5718                 break;
5719         }
5720
5721         if (rdev->flags & RADEON_IS_IGP)
5722                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5723
5724         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5725         if (fusion_platform)
5726                 data &= ~MULTI_PIF;
5727         else
5728                 data |= MULTI_PIF;
5729         if (data != orig)
5730                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5731
5732         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5733         if (fusion_platform)
5734                 data &= ~MULTI_PIF;
5735         else
5736                 data |= MULTI_PIF;
5737         if (data != orig)
5738                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5739
5740         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5741         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5742         if (!disable_l0s) {
5743                 if (rdev->family >= CHIP_BARTS)
5744                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5745                 else
5746                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5747         }
5748
5749         if (!disable_l1) {
5750                 if (rdev->family >= CHIP_BARTS)
5751                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5752                 else
5753                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5754
5755                 if (!disable_plloff_in_l1) {
5756                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5757                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5758                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5759                         if (data != orig)
5760                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5761
5762                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5763                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5764                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5765                         if (data != orig)
5766                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5767
5768                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5769                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5770                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5771                         if (data != orig)
5772                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5773
5774                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5775                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5776                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5777                         if (data != orig)
5778                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5779
5780                         if (rdev->family >= CHIP_BARTS) {
5781                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5782                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5783                                 data |= PLL_RAMP_UP_TIME_0(4);
5784                                 if (data != orig)
5785                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5786
5787                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5788                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5789                                 data |= PLL_RAMP_UP_TIME_1(4);
5790                                 if (data != orig)
5791                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5792
5793                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5794                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5795                                 data |= PLL_RAMP_UP_TIME_0(4);
5796                                 if (data != orig)
5797                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5798
5799                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5800                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5801                                 data |= PLL_RAMP_UP_TIME_1(4);
5802                                 if (data != orig)
5803                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5804                         }
5805
5806                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5807                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5808                         data |= LC_DYN_LANES_PWR_STATE(3);
5809                         if (data != orig)
5810                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5811
5812                         if (rdev->family >= CHIP_BARTS) {
5813                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5814                                 data &= ~LS2_EXIT_TIME_MASK;
5815                                 data |= LS2_EXIT_TIME(1);
5816                                 if (data != orig)
5817                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5818
5819                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5820                                 data &= ~LS2_EXIT_TIME_MASK;
5821                                 data |= LS2_EXIT_TIME(1);
5822                                 if (data != orig)
5823                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5824                         }
5825                 }
5826         }
5827
5828         /* evergreen parts only */
5829         if (rdev->family < CHIP_BARTS)
5830                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5831
5832         if (pcie_lc_cntl != pcie_lc_cntl_old)
5833                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5834 }