kernel/drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include "radeon_audio.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 /*
  39  * Indirect registers accessor
  40  */
  41 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  42 {
  43         unsigned long flags;
  44         u32 r;
  45
  46         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  47         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  48         r = RREG32(EVERGREEN_CG_IND_DATA);
  49         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  50         return r;
  51 }
  52
  53 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  54 {
  55         unsigned long flags;
  56
  57         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  58         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  59         WREG32(EVERGREEN_CG_IND_DATA, (v));
  60         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  61 }
  62
  63 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  64 {
  65         unsigned long flags;
  66         u32 r;
  67
  68         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  69         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  70         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  71         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  72         return r;
  73 }
  74
  75 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  76 {
  77         unsigned long flags;
  78
  79         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  80         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  81         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  82         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  83 }
  84
  85 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  86 {
  87         unsigned long flags;
  88         u32 r;
  89
  90         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  91         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  92         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
  93         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  94         return r;
  95 }
  96
  97 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  98 {
  99         unsigned long flags;
 100
 101         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 102         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 103         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 104         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 105 }
 106
 107 static const u32 crtc_offsets[6] =
 108 {
 109         EVERGREEN_CRTC0_REGISTER_OFFSET,
 110         EVERGREEN_CRTC1_REGISTER_OFFSET,
 111         EVERGREEN_CRTC2_REGISTER_OFFSET,
 112         EVERGREEN_CRTC3_REGISTER_OFFSET,
 113         EVERGREEN_CRTC4_REGISTER_OFFSET,
 114         EVERGREEN_CRTC5_REGISTER_OFFSET
 115 };
 116
 117 #include "clearstate_evergreen.h"
 118
 119 static const u32 sumo_rlc_save_restore_register_list[] =
 120 {
 121         0x98fc,
 122         0x9830,
 123         0x9834,
 124         0x9838,
 125         0x9870,
 126         0x9874,
 127         0x8a14,
 128         0x8b24,
 129         0x8bcc,
 130         0x8b10,
 131         0x8d00,
 132         0x8d04,
 133         0x8c00,
 134         0x8c04,
 135         0x8c08,
 136         0x8c0c,
 137         0x8d8c,
 138         0x8c20,
 139         0x8c24,
 140         0x8c28,
 141         0x8c18,
 142         0x8c1c,
 143         0x8cf0,
 144         0x8e2c,
 145         0x8e38,
 146         0x8c30,
 147         0x9508,
 148         0x9688,
 149         0x9608,
 150         0x960c,
 151         0x9610,
 152         0x9614,
 153         0x88c4,
 154         0x88d4,
 155         0xa008,
 156         0x900c,
 157         0x9100,
 158         0x913c,
 159         0x98f8,
 160         0x98f4,
 161         0x9b7c,
 162         0x3f8c,
 163         0x8950,
 164         0x8954,
 165         0x8a18,
 166         0x8b28,
 167         0x9144,
 168         0x9148,
 169         0x914c,
 170         0x3f90,
 171         0x3f94,
 172         0x915c,
 173         0x9160,
 174         0x9178,
 175         0x917c,
 176         0x9180,
 177         0x918c,
 178         0x9190,
 179         0x9194,
 180         0x9198,
 181         0x919c,
 182         0x91a8,
 183         0x91ac,
 184         0x91b0,
 185         0x91b4,
 186         0x91b8,
 187         0x91c4,
 188         0x91c8,
 189         0x91cc,
 190         0x91d0,
 191         0x91d4,
 192         0x91e0,
 193         0x91e4,
 194         0x91ec,
 195         0x91f0,
 196         0x91f4,
 197         0x9200,
 198         0x9204,
 199         0x929c,
 200         0x9150,
 201         0x802c,
 202 };
 203
 204 static void evergreen_gpu_init(struct radeon_device *rdev);
 205 void evergreen_fini(struct radeon_device *rdev);
 206 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 207 void evergreen_program_aspm(struct radeon_device *rdev);
 208 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 209                                      int ring, u32 cp_int_cntl);
 210 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 211                                    u32 status, u32 addr);
 212 void cik_init_cp_pg_table(struct radeon_device *rdev);
 213
 214 extern u32 si_get_csb_size(struct radeon_device *rdev);
 215 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 216 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 217 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 218 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 219
 220 static const u32 evergreen_golden_registers[] =
 221 {
 222         0x3f90, 0xffff0000, 0xff000000,
 223         0x9148, 0xffff0000, 0xff000000,
 224         0x3f94, 0xffff0000, 0xff000000,
 225         0x914c, 0xffff0000, 0xff000000,
 226         0x9b7c, 0xffffffff, 0x00000000,
 227         0x8a14, 0xffffffff, 0x00000007,
 228         0x8b10, 0xffffffff, 0x00000000,
 229         0x960c, 0xffffffff, 0x54763210,
 230         0x88c4, 0xffffffff, 0x000000c2,
 231         0x88d4, 0xffffffff, 0x00000010,
 232         0x8974, 0xffffffff, 0x00000000,
 233         0xc78, 0x00000080, 0x00000080,
 234         0x5eb4, 0xffffffff, 0x00000002,
 235         0x5e78, 0xffffffff, 0x001000f0,
 236         0x6104, 0x01000300, 0x00000000,
 237         0x5bc0, 0x00300000, 0x00000000,
 238         0x7030, 0xffffffff, 0x00000011,
 239         0x7c30, 0xffffffff, 0x00000011,
 240         0x10830, 0xffffffff, 0x00000011,
 241         0x11430, 0xffffffff, 0x00000011,
 242         0x12030, 0xffffffff, 0x00000011,
 243         0x12c30, 0xffffffff, 0x00000011,
 244         0xd02c, 0xffffffff, 0x08421000,
 245         0x240c, 0xffffffff, 0x00000380,
 246         0x8b24, 0xffffffff, 0x00ff0fff,
 247         0x28a4c, 0x06000000, 0x06000000,
 248         0x10c, 0x00000001, 0x00000001,
 249         0x8d00, 0xffffffff, 0x100e4848,
 250         0x8d04, 0xffffffff, 0x00164745,
 251         0x8c00, 0xffffffff, 0xe4000003,
 252         0x8c04, 0xffffffff, 0x40600060,
 253         0x8c08, 0xffffffff, 0x001c001c,
 254         0x8cf0, 0xffffffff, 0x08e00620,
 255         0x8c20, 0xffffffff, 0x00800080,
 256         0x8c24, 0xffffffff, 0x00800080,
 257         0x8c18, 0xffffffff, 0x20202078,
 258         0x8c1c, 0xffffffff, 0x00001010,
 259         0x28350, 0xffffffff, 0x00000000,
 260         0xa008, 0xffffffff, 0x00010000,
 261         0x5c4, 0xffffffff, 0x00000001,
 262         0x9508, 0xffffffff, 0x00000002,
 263         0x913c, 0x0000000f, 0x0000000a
 264 };
 265
 266 static const u32 evergreen_golden_registers2[] =
 267 {
 268         0x2f4c, 0xffffffff, 0x00000000,
 269         0x54f4, 0xffffffff, 0x00000000,
 270         0x54f0, 0xffffffff, 0x00000000,
 271         0x5498, 0xffffffff, 0x00000000,
 272         0x549c, 0xffffffff, 0x00000000,
 273         0x5494, 0xffffffff, 0x00000000,
 274         0x53cc, 0xffffffff, 0x00000000,
 275         0x53c8, 0xffffffff, 0x00000000,
 276         0x53c4, 0xffffffff, 0x00000000,
 277         0x53c0, 0xffffffff, 0x00000000,
 278         0x53bc, 0xffffffff, 0x00000000,
 279         0x53b8, 0xffffffff, 0x00000000,
 280         0x53b4, 0xffffffff, 0x00000000,
 281         0x53b0, 0xffffffff, 0x00000000
 282 };
 283
 284 static const u32 cypress_mgcg_init[] =
 285 {
 286         0x802c, 0xffffffff, 0xc0000000,
 287         0x5448, 0xffffffff, 0x00000100,
 288         0x55e4, 0xffffffff, 0x00000100,
 289         0x160c, 0xffffffff, 0x00000100,
 290         0x5644, 0xffffffff, 0x00000100,
 291         0xc164, 0xffffffff, 0x00000100,
 292         0x8a18, 0xffffffff, 0x00000100,
 293         0x897c, 0xffffffff, 0x06000100,
 294         0x8b28, 0xffffffff, 0x00000100,
 295         0x9144, 0xffffffff, 0x00000100,
 296         0x9a60, 0xffffffff, 0x00000100,
 297         0x9868, 0xffffffff, 0x00000100,
 298         0x8d58, 0xffffffff, 0x00000100,
 299         0x9510, 0xffffffff, 0x00000100,
 300         0x949c, 0xffffffff, 0x00000100,
 301         0x9654, 0xffffffff, 0x00000100,
 302         0x9030, 0xffffffff, 0x00000100,
 303         0x9034, 0xffffffff, 0x00000100,
 304         0x9038, 0xffffffff, 0x00000100,
 305         0x903c, 0xffffffff, 0x00000100,
 306         0x9040, 0xffffffff, 0x00000100,
 307         0xa200, 0xffffffff, 0x00000100,
 308         0xa204, 0xffffffff, 0x00000100,
 309         0xa208, 0xffffffff, 0x00000100,
 310         0xa20c, 0xffffffff, 0x00000100,
 311         0x971c, 0xffffffff, 0x00000100,
 312         0x977c, 0xffffffff, 0x00000100,
 313         0x3f80, 0xffffffff, 0x00000100,
 314         0xa210, 0xffffffff, 0x00000100,
 315         0xa214, 0xffffffff, 0x00000100,
 316         0x4d8, 0xffffffff, 0x00000100,
 317         0x9784, 0xffffffff, 0x00000100,
 318         0x9698, 0xffffffff, 0x00000100,
 319         0x4d4, 0xffffffff, 0x00000200,
 320         0x30cc, 0xffffffff, 0x00000100,
 321         0xd0c0, 0xffffffff, 0xff000100,
 322         0x802c, 0xffffffff, 0x40000000,
 323         0x915c, 0xffffffff, 0x00010000,
 324         0x9160, 0xffffffff, 0x00030002,
 325         0x9178, 0xffffffff, 0x00070000,
 326         0x917c, 0xffffffff, 0x00030002,
 327         0x9180, 0xffffffff, 0x00050004,
 328         0x918c, 0xffffffff, 0x00010006,
 329         0x9190, 0xffffffff, 0x00090008,
 330         0x9194, 0xffffffff, 0x00070000,
 331         0x9198, 0xffffffff, 0x00030002,
 332         0x919c, 0xffffffff, 0x00050004,
 333         0x91a8, 0xffffffff, 0x00010006,
 334         0x91ac, 0xffffffff, 0x00090008,
 335         0x91b0, 0xffffffff, 0x00070000,
 336         0x91b4, 0xffffffff, 0x00030002,
 337         0x91b8, 0xffffffff, 0x00050004,
 338         0x91c4, 0xffffffff, 0x00010006,
 339         0x91c8, 0xffffffff, 0x00090008,
 340         0x91cc, 0xffffffff, 0x00070000,
 341         0x91d0, 0xffffffff, 0x00030002,
 342         0x91d4, 0xffffffff, 0x00050004,
 343         0x91e0, 0xffffffff, 0x00010006,
 344         0x91e4, 0xffffffff, 0x00090008,
 345         0x91e8, 0xffffffff, 0x00000000,
 346         0x91ec, 0xffffffff, 0x00070000,
 347         0x91f0, 0xffffffff, 0x00030002,
 348         0x91f4, 0xffffffff, 0x00050004,
 349         0x9200, 0xffffffff, 0x00010006,
 350         0x9204, 0xffffffff, 0x00090008,
 351         0x9208, 0xffffffff, 0x00070000,
 352         0x920c, 0xffffffff, 0x00030002,
 353         0x9210, 0xffffffff, 0x00050004,
 354         0x921c, 0xffffffff, 0x00010006,
 355         0x9220, 0xffffffff, 0x00090008,
 356         0x9224, 0xffffffff, 0x00070000,
 357         0x9228, 0xffffffff, 0x00030002,
 358         0x922c, 0xffffffff, 0x00050004,
 359         0x9238, 0xffffffff, 0x00010006,
 360         0x923c, 0xffffffff, 0x00090008,
 361         0x9240, 0xffffffff, 0x00070000,
 362         0x9244, 0xffffffff, 0x00030002,
 363         0x9248, 0xffffffff, 0x00050004,
 364         0x9254, 0xffffffff, 0x00010006,
 365         0x9258, 0xffffffff, 0x00090008,
 366         0x925c, 0xffffffff, 0x00070000,
 367         0x9260, 0xffffffff, 0x00030002,
 368         0x9264, 0xffffffff, 0x00050004,
 369         0x9270, 0xffffffff, 0x00010006,
 370         0x9274, 0xffffffff, 0x00090008,
 371         0x9278, 0xffffffff, 0x00070000,
 372         0x927c, 0xffffffff, 0x00030002,
 373         0x9280, 0xffffffff, 0x00050004,
 374         0x928c, 0xffffffff, 0x00010006,
 375         0x9290, 0xffffffff, 0x00090008,
 376         0x9294, 0xffffffff, 0x00000000,
 377         0x929c, 0xffffffff, 0x00000001,
 378         0x802c, 0xffffffff, 0x40010000,
 379         0x915c, 0xffffffff, 0x00010000,
 380         0x9160, 0xffffffff, 0x00030002,
 381         0x9178, 0xffffffff, 0x00070000,
 382         0x917c, 0xffffffff, 0x00030002,
 383         0x9180, 0xffffffff, 0x00050004,
 384         0x918c, 0xffffffff, 0x00010006,
 385         0x9190, 0xffffffff, 0x00090008,
 386         0x9194, 0xffffffff, 0x00070000,
 387         0x9198, 0xffffffff, 0x00030002,
 388         0x919c, 0xffffffff, 0x00050004,
 389         0x91a8, 0xffffffff, 0x00010006,
 390         0x91ac, 0xffffffff, 0x00090008,
 391         0x91b0, 0xffffffff, 0x00070000,
 392         0x91b4, 0xffffffff, 0x00030002,
 393         0x91b8, 0xffffffff, 0x00050004,
 394         0x91c4, 0xffffffff, 0x00010006,
 395         0x91c8, 0xffffffff, 0x00090008,
 396         0x91cc, 0xffffffff, 0x00070000,
 397         0x91d0, 0xffffffff, 0x00030002,
 398         0x91d4, 0xffffffff, 0x00050004,
 399         0x91e0, 0xffffffff, 0x00010006,
 400         0x91e4, 0xffffffff, 0x00090008,
 401         0x91e8, 0xffffffff, 0x00000000,
 402         0x91ec, 0xffffffff, 0x00070000,
 403         0x91f0, 0xffffffff, 0x00030002,
 404         0x91f4, 0xffffffff, 0x00050004,
 405         0x9200, 0xffffffff, 0x00010006,
 406         0x9204, 0xffffffff, 0x00090008,
 407         0x9208, 0xffffffff, 0x00070000,
 408         0x920c, 0xffffffff, 0x00030002,
 409         0x9210, 0xffffffff, 0x00050004,
 410         0x921c, 0xffffffff, 0x00010006,
 411         0x9220, 0xffffffff, 0x00090008,
 412         0x9224, 0xffffffff, 0x00070000,
 413         0x9228, 0xffffffff, 0x00030002,
 414         0x922c, 0xffffffff, 0x00050004,
 415         0x9238, 0xffffffff, 0x00010006,
 416         0x923c, 0xffffffff, 0x00090008,
 417         0x9240, 0xffffffff, 0x00070000,
 418         0x9244, 0xffffffff, 0x00030002,
 419         0x9248, 0xffffffff, 0x00050004,
 420         0x9254, 0xffffffff, 0x00010006,
 421         0x9258, 0xffffffff, 0x00090008,
 422         0x925c, 0xffffffff, 0x00070000,
 423         0x9260, 0xffffffff, 0x00030002,
 424         0x9264, 0xffffffff, 0x00050004,
 425         0x9270, 0xffffffff, 0x00010006,
 426         0x9274, 0xffffffff, 0x00090008,
 427         0x9278, 0xffffffff, 0x00070000,
 428         0x927c, 0xffffffff, 0x00030002,
 429         0x9280, 0xffffffff, 0x00050004,
 430         0x928c, 0xffffffff, 0x00010006,
 431         0x9290, 0xffffffff, 0x00090008,
 432         0x9294, 0xffffffff, 0x00000000,
 433         0x929c, 0xffffffff, 0x00000001,
 434         0x802c, 0xffffffff, 0xc0000000
 435 };
 436
 437 static const u32 redwood_mgcg_init[] =
 438 {
 439         0x802c, 0xffffffff, 0xc0000000,
 440         0x5448, 0xffffffff, 0x00000100,
 441         0x55e4, 0xffffffff, 0x00000100,
 442         0x160c, 0xffffffff, 0x00000100,
 443         0x5644, 0xffffffff, 0x00000100,
 444         0xc164, 0xffffffff, 0x00000100,
 445         0x8a18, 0xffffffff, 0x00000100,
 446         0x897c, 0xffffffff, 0x06000100,
 447         0x8b28, 0xffffffff, 0x00000100,
 448         0x9144, 0xffffffff, 0x00000100,
 449         0x9a60, 0xffffffff, 0x00000100,
 450         0x9868, 0xffffffff, 0x00000100,
 451         0x8d58, 0xffffffff, 0x00000100,
 452         0x9510, 0xffffffff, 0x00000100,
 453         0x949c, 0xffffffff, 0x00000100,
 454         0x9654, 0xffffffff, 0x00000100,
 455         0x9030, 0xffffffff, 0x00000100,
 456         0x9034, 0xffffffff, 0x00000100,
 457         0x9038, 0xffffffff, 0x00000100,
 458         0x903c, 0xffffffff, 0x00000100,
 459         0x9040, 0xffffffff, 0x00000100,
 460         0xa200, 0xffffffff, 0x00000100,
 461         0xa204, 0xffffffff, 0x00000100,
 462         0xa208, 0xffffffff, 0x00000100,
 463         0xa20c, 0xffffffff, 0x00000100,
 464         0x971c, 0xffffffff, 0x00000100,
 465         0x977c, 0xffffffff, 0x00000100,
 466         0x3f80, 0xffffffff, 0x00000100,
 467         0xa210, 0xffffffff, 0x00000100,
 468         0xa214, 0xffffffff, 0x00000100,
 469         0x4d8, 0xffffffff, 0x00000100,
 470         0x9784, 0xffffffff, 0x00000100,
 471         0x9698, 0xffffffff, 0x00000100,
 472         0x4d4, 0xffffffff, 0x00000200,
 473         0x30cc, 0xffffffff, 0x00000100,
 474         0xd0c0, 0xffffffff, 0xff000100,
 475         0x802c, 0xffffffff, 0x40000000,
 476         0x915c, 0xffffffff, 0x00010000,
 477         0x9160, 0xffffffff, 0x00030002,
 478         0x9178, 0xffffffff, 0x00070000,
 479         0x917c, 0xffffffff, 0x00030002,
 480         0x9180, 0xffffffff, 0x00050004,
 481         0x918c, 0xffffffff, 0x00010006,
 482         0x9190, 0xffffffff, 0x00090008,
 483         0x9194, 0xffffffff, 0x00070000,
 484         0x9198, 0xffffffff, 0x00030002,
 485         0x919c, 0xffffffff, 0x00050004,
 486         0x91a8, 0xffffffff, 0x00010006,
 487         0x91ac, 0xffffffff, 0x00090008,
 488         0x91b0, 0xffffffff, 0x00070000,
 489         0x91b4, 0xffffffff, 0x00030002,
 490         0x91b8, 0xffffffff, 0x00050004,
 491         0x91c4, 0xffffffff, 0x00010006,
 492         0x91c8, 0xffffffff, 0x00090008,
 493         0x91cc, 0xffffffff, 0x00070000,
 494         0x91d0, 0xffffffff, 0x00030002,
 495         0x91d4, 0xffffffff, 0x00050004,
 496         0x91e0, 0xffffffff, 0x00010006,
 497         0x91e4, 0xffffffff, 0x00090008,
 498         0x91e8, 0xffffffff, 0x00000000,
 499         0x91ec, 0xffffffff, 0x00070000,
 500         0x91f0, 0xffffffff, 0x00030002,
 501         0x91f4, 0xffffffff, 0x00050004,
 502         0x9200, 0xffffffff, 0x00010006,
 503         0x9204, 0xffffffff, 0x00090008,
 504         0x9294, 0xffffffff, 0x00000000,
 505         0x929c, 0xffffffff, 0x00000001,
 506         0x802c, 0xffffffff, 0xc0000000
 507 };
 508
 509 static const u32 cedar_golden_registers[] =
 510 {
 511         0x3f90, 0xffff0000, 0xff000000,
 512         0x9148, 0xffff0000, 0xff000000,
 513         0x3f94, 0xffff0000, 0xff000000,
 514         0x914c, 0xffff0000, 0xff000000,
 515         0x9b7c, 0xffffffff, 0x00000000,
 516         0x8a14, 0xffffffff, 0x00000007,
 517         0x8b10, 0xffffffff, 0x00000000,
 518         0x960c, 0xffffffff, 0x54763210,
 519         0x88c4, 0xffffffff, 0x000000c2,
 520         0x88d4, 0xffffffff, 0x00000000,
 521         0x8974, 0xffffffff, 0x00000000,
 522         0xc78, 0x00000080, 0x00000080,
 523         0x5eb4, 0xffffffff, 0x00000002,
 524         0x5e78, 0xffffffff, 0x001000f0,
 525         0x6104, 0x01000300, 0x00000000,
 526         0x5bc0, 0x00300000, 0x00000000,
 527         0x7030, 0xffffffff, 0x00000011,
 528         0x7c30, 0xffffffff, 0x00000011,
 529         0x10830, 0xffffffff, 0x00000011,
 530         0x11430, 0xffffffff, 0x00000011,
 531         0xd02c, 0xffffffff, 0x08421000,
 532         0x240c, 0xffffffff, 0x00000380,
 533         0x8b24, 0xffffffff, 0x00ff0fff,
 534         0x28a4c, 0x06000000, 0x06000000,
 535         0x10c, 0x00000001, 0x00000001,
 536         0x8d00, 0xffffffff, 0x100e4848,
 537         0x8d04, 0xffffffff, 0x00164745,
 538         0x8c00, 0xffffffff, 0xe4000003,
 539         0x8c04, 0xffffffff, 0x40600060,
 540         0x8c08, 0xffffffff, 0x001c001c,
 541         0x8cf0, 0xffffffff, 0x08e00410,
 542         0x8c20, 0xffffffff, 0x00800080,
 543         0x8c24, 0xffffffff, 0x00800080,
 544         0x8c18, 0xffffffff, 0x20202078,
 545         0x8c1c, 0xffffffff, 0x00001010,
 546         0x28350, 0xffffffff, 0x00000000,
 547         0xa008, 0xffffffff, 0x00010000,
 548         0x5c4, 0xffffffff, 0x00000001,
 549         0x9508, 0xffffffff, 0x00000002
 550 };
 551
 552 static const u32 cedar_mgcg_init[] =
 553 {
 554         0x802c, 0xffffffff, 0xc0000000,
 555         0x5448, 0xffffffff, 0x00000100,
 556         0x55e4, 0xffffffff, 0x00000100,
 557         0x160c, 0xffffffff, 0x00000100,
 558         0x5644, 0xffffffff, 0x00000100,
 559         0xc164, 0xffffffff, 0x00000100,
 560         0x8a18, 0xffffffff, 0x00000100,
 561         0x897c, 0xffffffff, 0x06000100,
 562         0x8b28, 0xffffffff, 0x00000100,
 563         0x9144, 0xffffffff, 0x00000100,
 564         0x9a60, 0xffffffff, 0x00000100,
 565         0x9868, 0xffffffff, 0x00000100,
 566         0x8d58, 0xffffffff, 0x00000100,
 567         0x9510, 0xffffffff, 0x00000100,
 568         0x949c, 0xffffffff, 0x00000100,
 569         0x9654, 0xffffffff, 0x00000100,
 570         0x9030, 0xffffffff, 0x00000100,
 571         0x9034, 0xffffffff, 0x00000100,
 572         0x9038, 0xffffffff, 0x00000100,
 573         0x903c, 0xffffffff, 0x00000100,
 574         0x9040, 0xffffffff, 0x00000100,
 575         0xa200, 0xffffffff, 0x00000100,
 576         0xa204, 0xffffffff, 0x00000100,
 577         0xa208, 0xffffffff, 0x00000100,
 578         0xa20c, 0xffffffff, 0x00000100,
 579         0x971c, 0xffffffff, 0x00000100,
 580         0x977c, 0xffffffff, 0x00000100,
 581         0x3f80, 0xffffffff, 0x00000100,
 582         0xa210, 0xffffffff, 0x00000100,
 583         0xa214, 0xffffffff, 0x00000100,
 584         0x4d8, 0xffffffff, 0x00000100,
 585         0x9784, 0xffffffff, 0x00000100,
 586         0x9698, 0xffffffff, 0x00000100,
 587         0x4d4, 0xffffffff, 0x00000200,
 588         0x30cc, 0xffffffff, 0x00000100,
 589         0xd0c0, 0xffffffff, 0xff000100,
 590         0x802c, 0xffffffff, 0x40000000,
 591         0x915c, 0xffffffff, 0x00010000,
 592         0x9178, 0xffffffff, 0x00050000,
 593         0x917c, 0xffffffff, 0x00030002,
 594         0x918c, 0xffffffff, 0x00010004,
 595         0x9190, 0xffffffff, 0x00070006,
 596         0x9194, 0xffffffff, 0x00050000,
 597         0x9198, 0xffffffff, 0x00030002,
 598         0x91a8, 0xffffffff, 0x00010004,
 599         0x91ac, 0xffffffff, 0x00070006,
 600         0x91e8, 0xffffffff, 0x00000000,
 601         0x9294, 0xffffffff, 0x00000000,
 602         0x929c, 0xffffffff, 0x00000001,
 603         0x802c, 0xffffffff, 0xc0000000
 604 };
 605
 606 static const u32 juniper_mgcg_init[] =
 607 {
 608         0x802c, 0xffffffff, 0xc0000000,
 609         0x5448, 0xffffffff, 0x00000100,
 610         0x55e4, 0xffffffff, 0x00000100,
 611         0x160c, 0xffffffff, 0x00000100,
 612         0x5644, 0xffffffff, 0x00000100,
 613         0xc164, 0xffffffff, 0x00000100,
 614         0x8a18, 0xffffffff, 0x00000100,
 615         0x897c, 0xffffffff, 0x06000100,
 616         0x8b28, 0xffffffff, 0x00000100,
 617         0x9144, 0xffffffff, 0x00000100,
 618         0x9a60, 0xffffffff, 0x00000100,
 619         0x9868, 0xffffffff, 0x00000100,
 620         0x8d58, 0xffffffff, 0x00000100,
 621         0x9510, 0xffffffff, 0x00000100,
 622         0x949c, 0xffffffff, 0x00000100,
 623         0x9654, 0xffffffff, 0x00000100,
 624         0x9030, 0xffffffff, 0x00000100,
 625         0x9034, 0xffffffff, 0x00000100,
 626         0x9038, 0xffffffff, 0x00000100,
 627         0x903c, 0xffffffff, 0x00000100,
 628         0x9040, 0xffffffff, 0x00000100,
 629         0xa200, 0xffffffff, 0x00000100,
 630         0xa204, 0xffffffff, 0x00000100,
 631         0xa208, 0xffffffff, 0x00000100,
 632         0xa20c, 0xffffffff, 0x00000100,
 633         0x971c, 0xffffffff, 0x00000100,
 634         0xd0c0, 0xffffffff, 0xff000100,
 635         0x802c, 0xffffffff, 0x40000000,
 636         0x915c, 0xffffffff, 0x00010000,
 637         0x9160, 0xffffffff, 0x00030002,
 638         0x9178, 0xffffffff, 0x00070000,
 639         0x917c, 0xffffffff, 0x00030002,
 640         0x9180, 0xffffffff, 0x00050004,
 641         0x918c, 0xffffffff, 0x00010006,
 642         0x9190, 0xffffffff, 0x00090008,
 643         0x9194, 0xffffffff, 0x00070000,
 644         0x9198, 0xffffffff, 0x00030002,
 645         0x919c, 0xffffffff, 0x00050004,
 646         0x91a8, 0xffffffff, 0x00010006,
 647         0x91ac, 0xffffffff, 0x00090008,
 648         0x91b0, 0xffffffff, 0x00070000,
 649         0x91b4, 0xffffffff, 0x00030002,
 650         0x91b8, 0xffffffff, 0x00050004,
 651         0x91c4, 0xffffffff, 0x00010006,
 652         0x91c8, 0xffffffff, 0x00090008,
 653         0x91cc, 0xffffffff, 0x00070000,
 654         0x91d0, 0xffffffff, 0x00030002,
 655         0x91d4, 0xffffffff, 0x00050004,
 656         0x91e0, 0xffffffff, 0x00010006,
 657         0x91e4, 0xffffffff, 0x00090008,
 658         0x91e8, 0xffffffff, 0x00000000,
 659         0x91ec, 0xffffffff, 0x00070000,
 660         0x91f0, 0xffffffff, 0x00030002,
 661         0x91f4, 0xffffffff, 0x00050004,
 662         0x9200, 0xffffffff, 0x00010006,
 663         0x9204, 0xffffffff, 0x00090008,
 664         0x9208, 0xffffffff, 0x00070000,
 665         0x920c, 0xffffffff, 0x00030002,
 666         0x9210, 0xffffffff, 0x00050004,
 667         0x921c, 0xffffffff, 0x00010006,
 668         0x9220, 0xffffffff, 0x00090008,
 669         0x9224, 0xffffffff, 0x00070000,
 670         0x9228, 0xffffffff, 0x00030002,
 671         0x922c, 0xffffffff, 0x00050004,
 672         0x9238, 0xffffffff, 0x00010006,
 673         0x923c, 0xffffffff, 0x00090008,
 674         0x9240, 0xffffffff, 0x00070000,
 675         0x9244, 0xffffffff, 0x00030002,
 676         0x9248, 0xffffffff, 0x00050004,
 677         0x9254, 0xffffffff, 0x00010006,
 678         0x9258, 0xffffffff, 0x00090008,
 679         0x925c, 0xffffffff, 0x00070000,
 680         0x9260, 0xffffffff, 0x00030002,
 681         0x9264, 0xffffffff, 0x00050004,
 682         0x9270, 0xffffffff, 0x00010006,
 683         0x9274, 0xffffffff, 0x00090008,
 684         0x9278, 0xffffffff, 0x00070000,
 685         0x927c, 0xffffffff, 0x00030002,
 686         0x9280, 0xffffffff, 0x00050004,
 687         0x928c, 0xffffffff, 0x00010006,
 688         0x9290, 0xffffffff, 0x00090008,
 689         0x9294, 0xffffffff, 0x00000000,
 690         0x929c, 0xffffffff, 0x00000001,
 691         0x802c, 0xffffffff, 0xc0000000,
 692         0x977c, 0xffffffff, 0x00000100,
 693         0x3f80, 0xffffffff, 0x00000100,
 694         0xa210, 0xffffffff, 0x00000100,
 695         0xa214, 0xffffffff, 0x00000100,
 696         0x4d8, 0xffffffff, 0x00000100,
 697         0x9784, 0xffffffff, 0x00000100,
 698         0x9698, 0xffffffff, 0x00000100,
 699         0x4d4, 0xffffffff, 0x00000200,
 700         0x30cc, 0xffffffff, 0x00000100,
 701         0x802c, 0xffffffff, 0xc0000000
 702 };
 703
 704 static const u32 supersumo_golden_registers[] =
 705 {
 706         0x5eb4, 0xffffffff, 0x00000002,
 707         0x5c4, 0xffffffff, 0x00000001,
 708         0x7030, 0xffffffff, 0x00000011,
 709         0x7c30, 0xffffffff, 0x00000011,
 710         0x6104, 0x01000300, 0x00000000,
 711         0x5bc0, 0x00300000, 0x00000000,
 712         0x8c04, 0xffffffff, 0x40600060,
 713         0x8c08, 0xffffffff, 0x001c001c,
 714         0x8c20, 0xffffffff, 0x00800080,
 715         0x8c24, 0xffffffff, 0x00800080,
 716         0x8c18, 0xffffffff, 0x20202078,
 717         0x8c1c, 0xffffffff, 0x00001010,
 718         0x918c, 0xffffffff, 0x00010006,
 719         0x91a8, 0xffffffff, 0x00010006,
 720         0x91c4, 0xffffffff, 0x00010006,
 721         0x91e0, 0xffffffff, 0x00010006,
 722         0x9200, 0xffffffff, 0x00010006,
 723         0x9150, 0xffffffff, 0x6e944040,
 724         0x917c, 0xffffffff, 0x00030002,
 725         0x9180, 0xffffffff, 0x00050004,
 726         0x9198, 0xffffffff, 0x00030002,
 727         0x919c, 0xffffffff, 0x00050004,
 728         0x91b4, 0xffffffff, 0x00030002,
 729         0x91b8, 0xffffffff, 0x00050004,
 730         0x91d0, 0xffffffff, 0x00030002,
 731         0x91d4, 0xffffffff, 0x00050004,
 732         0x91f0, 0xffffffff, 0x00030002,
 733         0x91f4, 0xffffffff, 0x00050004,
 734         0x915c, 0xffffffff, 0x00010000,
 735         0x9160, 0xffffffff, 0x00030002,
 736         0x3f90, 0xffff0000, 0xff000000,
 737         0x9178, 0xffffffff, 0x00070000,
 738         0x9194, 0xffffffff, 0x00070000,
 739         0x91b0, 0xffffffff, 0x00070000,
 740         0x91cc, 0xffffffff, 0x00070000,
 741         0x91ec, 0xffffffff, 0x00070000,
 742         0x9148, 0xffff0000, 0xff000000,
 743         0x9190, 0xffffffff, 0x00090008,
 744         0x91ac, 0xffffffff, 0x00090008,
 745         0x91c8, 0xffffffff, 0x00090008,
 746         0x91e4, 0xffffffff, 0x00090008,
 747         0x9204, 0xffffffff, 0x00090008,
 748         0x3f94, 0xffff0000, 0xff000000,
 749         0x914c, 0xffff0000, 0xff000000,
 750         0x929c, 0xffffffff, 0x00000001,
 751         0x8a18, 0xffffffff, 0x00000100,
 752         0x8b28, 0xffffffff, 0x00000100,
 753         0x9144, 0xffffffff, 0x00000100,
 754         0x5644, 0xffffffff, 0x00000100,
 755         0x9b7c, 0xffffffff, 0x00000000,
 756         0x8030, 0xffffffff, 0x0000100a,
 757         0x8a14, 0xffffffff, 0x00000007,
 758         0x8b24, 0xffffffff, 0x00ff0fff,
 759         0x8b10, 0xffffffff, 0x00000000,
 760         0x28a4c, 0x06000000, 0x06000000,
 761         0x4d8, 0xffffffff, 0x00000100,
 762         0x913c, 0xffff000f, 0x0100000a,
 763         0x960c, 0xffffffff, 0x54763210,
 764         0x88c4, 0xffffffff, 0x000000c2,
 765         0x88d4, 0xffffffff, 0x00000010,
 766         0x8974, 0xffffffff, 0x00000000,
 767         0xc78, 0x00000080, 0x00000080,
 768         0x5e78, 0xffffffff, 0x001000f0,
 769         0xd02c, 0xffffffff, 0x08421000,
 770         0xa008, 0xffffffff, 0x00010000,
 771         0x8d00, 0xffffffff, 0x100e4848,
 772         0x8d04, 0xffffffff, 0x00164745,
 773         0x8c00, 0xffffffff, 0xe4000003,
 774         0x8cf0, 0x1fffffff, 0x08e00620,
 775         0x28350, 0xffffffff, 0x00000000,
 776         0x9508, 0xffffffff, 0x00000002
 777 };
 778
 779 static const u32 sumo_golden_registers[] =
 780 {
 781         0x900c, 0x00ffffff, 0x0017071f,
 782         0x8c18, 0xffffffff, 0x10101060,
 783         0x8c1c, 0xffffffff, 0x00001010,
 784         0x8c30, 0x0000000f, 0x00000005,
 785         0x9688, 0x0000000f, 0x00000007
 786 };
 787
 788 static const u32 wrestler_golden_registers[] =
 789 {
 790         0x5eb4, 0xffffffff, 0x00000002,
 791         0x5c4, 0xffffffff, 0x00000001,
 792         0x7030, 0xffffffff, 0x00000011,
 793         0x7c30, 0xffffffff, 0x00000011,
 794         0x6104, 0x01000300, 0x00000000,
 795         0x5bc0, 0x00300000, 0x00000000,
 796         0x918c, 0xffffffff, 0x00010006,
 797         0x91a8, 0xffffffff, 0x00010006,
 798         0x9150, 0xffffffff, 0x6e944040,
 799         0x917c, 0xffffffff, 0x00030002,
 800         0x9198, 0xffffffff, 0x00030002,
 801         0x915c, 0xffffffff, 0x00010000,
 802         0x3f90, 0xffff0000, 0xff000000,
 803         0x9178, 0xffffffff, 0x00070000,
 804         0x9194, 0xffffffff, 0x00070000,
 805         0x9148, 0xffff0000, 0xff000000,
 806         0x9190, 0xffffffff, 0x00090008,
 807         0x91ac, 0xffffffff, 0x00090008,
 808         0x3f94, 0xffff0000, 0xff000000,
 809         0x914c, 0xffff0000, 0xff000000,
 810         0x929c, 0xffffffff, 0x00000001,
 811         0x8a18, 0xffffffff, 0x00000100,
 812         0x8b28, 0xffffffff, 0x00000100,
 813         0x9144, 0xffffffff, 0x00000100,
 814         0x9b7c, 0xffffffff, 0x00000000,
 815         0x8030, 0xffffffff, 0x0000100a,
 816         0x8a14, 0xffffffff, 0x00000001,
 817         0x8b24, 0xffffffff, 0x00ff0fff,
 818         0x8b10, 0xffffffff, 0x00000000,
 819         0x28a4c, 0x06000000, 0x06000000,
 820         0x4d8, 0xffffffff, 0x00000100,
 821         0x913c, 0xffff000f, 0x0100000a,
 822         0x960c, 0xffffffff, 0x54763210,
 823         0x88c4, 0xffffffff, 0x000000c2,
 824         0x88d4, 0xffffffff, 0x00000010,
 825         0x8974, 0xffffffff, 0x00000000,
 826         0xc78, 0x00000080, 0x00000080,
 827         0x5e78, 0xffffffff, 0x001000f0,
 828         0xd02c, 0xffffffff, 0x08421000,
 829         0xa008, 0xffffffff, 0x00010000,
 830         0x8d00, 0xffffffff, 0x100e4848,
 831         0x8d04, 0xffffffff, 0x00164745,
 832         0x8c00, 0xffffffff, 0xe4000003,
 833         0x8cf0, 0x1fffffff, 0x08e00410,
 834         0x28350, 0xffffffff, 0x00000000,
 835         0x9508, 0xffffffff, 0x00000002,
 836         0x900c, 0xffffffff, 0x0017071f,
 837         0x8c18, 0xffffffff, 0x10101060,
 838         0x8c1c, 0xffffffff, 0x00001010
 839 };
 840
 841 static const u32 barts_golden_registers[] =
 842 {
 843         0x5eb4, 0xffffffff, 0x00000002,
 844         0x5e78, 0x8f311ff1, 0x001000f0,
 845         0x3f90, 0xffff0000, 0xff000000,
 846         0x9148, 0xffff0000, 0xff000000,
 847         0x3f94, 0xffff0000, 0xff000000,
 848         0x914c, 0xffff0000, 0xff000000,
 849         0xc78, 0x00000080, 0x00000080,
 850         0xbd4, 0x70073777, 0x00010001,
 851         0xd02c, 0xbfffff1f, 0x08421000,
 852         0xd0b8, 0x03773777, 0x02011003,
 853         0x5bc0, 0x00200000, 0x50100000,
 854         0x98f8, 0x33773777, 0x02011003,
 855         0x98fc, 0xffffffff, 0x76543210,
 856         0x7030, 0x31000311, 0x00000011,
 857         0x2f48, 0x00000007, 0x02011003,
 858         0x6b28, 0x00000010, 0x00000012,
 859         0x7728, 0x00000010, 0x00000012,
 860         0x10328, 0x00000010, 0x00000012,
 861         0x10f28, 0x00000010, 0x00000012,
 862         0x11b28, 0x00000010, 0x00000012,
 863         0x12728, 0x00000010, 0x00000012,
 864         0x240c, 0x000007ff, 0x00000380,
 865         0x8a14, 0xf000001f, 0x00000007,
 866         0x8b24, 0x3fff3fff, 0x00ff0fff,
 867         0x8b10, 0x0000ff0f, 0x00000000,
 868         0x28a4c, 0x07ffffff, 0x06000000,
 869         0x10c, 0x00000001, 0x00010003,
 870         0xa02c, 0xffffffff, 0x0000009b,
 871         0x913c, 0x0000000f, 0x0100000a,
 872         0x8d00, 0xffff7f7f, 0x100e4848,
 873         0x8d04, 0x00ffffff, 0x00164745,
 874         0x8c00, 0xfffc0003, 0xe4000003,
 875         0x8c04, 0xf8ff00ff, 0x40600060,
 876         0x8c08, 0x00ff00ff, 0x001c001c,
 877         0x8cf0, 0x1fff1fff, 0x08e00620,
 878         0x8c20, 0x0fff0fff, 0x00800080,
 879         0x8c24, 0x0fff0fff, 0x00800080,
 880         0x8c18, 0xffffffff, 0x20202078,
 881         0x8c1c, 0x0000ffff, 0x00001010,
 882         0x28350, 0x00000f01, 0x00000000,
 883         0x9508, 0x3700001f, 0x00000002,
 884         0x960c, 0xffffffff, 0x54763210,
 885         0x88c4, 0x001f3ae3, 0x000000c2,
 886         0x88d4, 0x0000001f, 0x00000010,
 887         0x8974, 0xffffffff, 0x00000000
 888 };
 889
 890 static const u32 turks_golden_registers[] =
 891 {
 892         0x5eb4, 0xffffffff, 0x00000002,
 893         0x5e78, 0x8f311ff1, 0x001000f0,
 894         0x8c8, 0x00003000, 0x00001070,
 895         0x8cc, 0x000fffff, 0x00040035,
 896         0x3f90, 0xffff0000, 0xfff00000,
 897         0x9148, 0xffff0000, 0xfff00000,
 898         0x3f94, 0xffff0000, 0xfff00000,
 899         0x914c, 0xffff0000, 0xfff00000,
 900         0xc78, 0x00000080, 0x00000080,
 901         0xbd4, 0x00073007, 0x00010002,
 902         0xd02c, 0xbfffff1f, 0x08421000,
 903         0xd0b8, 0x03773777, 0x02010002,
 904         0x5bc0, 0x00200000, 0x50100000,
 905         0x98f8, 0x33773777, 0x00010002,
 906         0x98fc, 0xffffffff, 0x33221100,
 907         0x7030, 0x31000311, 0x00000011,
 908         0x2f48, 0x33773777, 0x00010002,
 909         0x6b28, 0x00000010, 0x00000012,
 910         0x7728, 0x00000010, 0x00000012,
 911         0x10328, 0x00000010, 0x00000012,
 912         0x10f28, 0x00000010, 0x00000012,
 913         0x11b28, 0x00000010, 0x00000012,
 914         0x12728, 0x00000010, 0x00000012,
 915         0x240c, 0x000007ff, 0x00000380,
 916         0x8a14, 0xf000001f, 0x00000007,
 917         0x8b24, 0x3fff3fff, 0x00ff0fff,
 918         0x8b10, 0x0000ff0f, 0x00000000,
 919         0x28a4c, 0x07ffffff, 0x06000000,
 920         0x10c, 0x00000001, 0x00010003,
 921         0xa02c, 0xffffffff, 0x0000009b,
 922         0x913c, 0x0000000f, 0x0100000a,
 923         0x8d00, 0xffff7f7f, 0x100e4848,
 924         0x8d04, 0x00ffffff, 0x00164745,
 925         0x8c00, 0xfffc0003, 0xe4000003,
 926         0x8c04, 0xf8ff00ff, 0x40600060,
 927         0x8c08, 0x00ff00ff, 0x001c001c,
 928         0x8cf0, 0x1fff1fff, 0x08e00410,
 929         0x8c20, 0x0fff0fff, 0x00800080,
 930         0x8c24, 0x0fff0fff, 0x00800080,
 931         0x8c18, 0xffffffff, 0x20202078,
 932         0x8c1c, 0x0000ffff, 0x00001010,
 933         0x28350, 0x00000f01, 0x00000000,
 934         0x9508, 0x3700001f, 0x00000002,
 935         0x960c, 0xffffffff, 0x54763210,
 936         0x88c4, 0x001f3ae3, 0x000000c2,
 937         0x88d4, 0x0000001f, 0x00000010,
 938         0x8974, 0xffffffff, 0x00000000
 939 };
 940
 941 static const u32 caicos_golden_registers[] =
 942 {
 943         0x5eb4, 0xffffffff, 0x00000002,
 944         0x5e78, 0x8f311ff1, 0x001000f0,
 945         0x8c8, 0x00003420, 0x00001450,
 946         0x8cc, 0x000fffff, 0x00040035,
 947         0x3f90, 0xffff0000, 0xfffc0000,
 948         0x9148, 0xffff0000, 0xfffc0000,
 949         0x3f94, 0xffff0000, 0xfffc0000,
 950         0x914c, 0xffff0000, 0xfffc0000,
 951         0xc78, 0x00000080, 0x00000080,
 952         0xbd4, 0x00073007, 0x00010001,
 953         0xd02c, 0xbfffff1f, 0x08421000,
 954         0xd0b8, 0x03773777, 0x02010001,
 955         0x5bc0, 0x00200000, 0x50100000,
 956         0x98f8, 0x33773777, 0x02010001,
 957         0x98fc, 0xffffffff, 0x33221100,
 958         0x7030, 0x31000311, 0x00000011,
 959         0x2f48, 0x33773777, 0x02010001,
 960         0x6b28, 0x00000010, 0x00000012,
 961         0x7728, 0x00000010, 0x00000012,
 962         0x10328, 0x00000010, 0x00000012,
 963         0x10f28, 0x00000010, 0x00000012,
 964         0x11b28, 0x00000010, 0x00000012,
 965         0x12728, 0x00000010, 0x00000012,
 966         0x240c, 0x000007ff, 0x00000380,
 967         0x8a14, 0xf000001f, 0x00000001,
 968         0x8b24, 0x3fff3fff, 0x00ff0fff,
 969         0x8b10, 0x0000ff0f, 0x00000000,
 970         0x28a4c, 0x07ffffff, 0x06000000,
 971         0x10c, 0x00000001, 0x00010003,
 972         0xa02c, 0xffffffff, 0x0000009b,
 973         0x913c, 0x0000000f, 0x0100000a,
 974         0x8d00, 0xffff7f7f, 0x100e4848,
 975         0x8d04, 0x00ffffff, 0x00164745,
 976         0x8c00, 0xfffc0003, 0xe4000003,
 977         0x8c04, 0xf8ff00ff, 0x40600060,
 978         0x8c08, 0x00ff00ff, 0x001c001c,
 979         0x8cf0, 0x1fff1fff, 0x08e00410,
 980         0x8c20, 0x0fff0fff, 0x00800080,
 981         0x8c24, 0x0fff0fff, 0x00800080,
 982         0x8c18, 0xffffffff, 0x20202078,
 983         0x8c1c, 0x0000ffff, 0x00001010,
 984         0x28350, 0x00000f01, 0x00000000,
 985         0x9508, 0x3700001f, 0x00000002,
 986         0x960c, 0xffffffff, 0x54763210,
 987         0x88c4, 0x001f3ae3, 0x000000c2,
 988         0x88d4, 0x0000001f, 0x00000010,
 989         0x8974, 0xffffffff, 0x00000000
 990 };
 991
 992 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 993 {
 994         switch (rdev->family) {
 995         case CHIP_CYPRESS:
 996         case CHIP_HEMLOCK:
 997                 radeon_program_register_sequence(rdev,
 998                                                  evergreen_golden_registers,
 999                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000                 radeon_program_register_sequence(rdev,
1001                                                  evergreen_golden_registers2,
1002                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003                 radeon_program_register_sequence(rdev,
1004                                                  cypress_mgcg_init,
1005                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006                 break;
1007         case CHIP_JUNIPER:
1008                 radeon_program_register_sequence(rdev,
1009                                                  evergreen_golden_registers,
1010                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011                 radeon_program_register_sequence(rdev,
1012                                                  evergreen_golden_registers2,
1013                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014                 radeon_program_register_sequence(rdev,
1015                                                  juniper_mgcg_init,
1016                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017                 break;
1018         case CHIP_REDWOOD:
1019                 radeon_program_register_sequence(rdev,
1020                                                  evergreen_golden_registers,
1021                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022                 radeon_program_register_sequence(rdev,
1023                                                  evergreen_golden_registers2,
1024                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025                 radeon_program_register_sequence(rdev,
1026                                                  redwood_mgcg_init,
1027                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028                 break;
1029         case CHIP_CEDAR:
1030                 radeon_program_register_sequence(rdev,
1031                                                  cedar_golden_registers,
1032                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1033                 radeon_program_register_sequence(rdev,
1034                                                  evergreen_golden_registers2,
1035                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036                 radeon_program_register_sequence(rdev,
1037                                                  cedar_mgcg_init,
1038                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039                 break;
1040         case CHIP_PALM:
1041                 radeon_program_register_sequence(rdev,
1042                                                  wrestler_golden_registers,
1043                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044                 break;
1045         case CHIP_SUMO:
1046                 radeon_program_register_sequence(rdev,
1047                                                  supersumo_golden_registers,
1048                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049                 break;
1050         case CHIP_SUMO2:
1051                 radeon_program_register_sequence(rdev,
1052                                                  supersumo_golden_registers,
1053                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054                 radeon_program_register_sequence(rdev,
1055                                                  sumo_golden_registers,
1056                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1057                 break;
1058         case CHIP_BARTS:
1059                 radeon_program_register_sequence(rdev,
1060                                                  barts_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1062                 break;
1063         case CHIP_TURKS:
1064                 radeon_program_register_sequence(rdev,
1065                                                  turks_golden_registers,
1066                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1067                 break;
1068         case CHIP_CAICOS:
1069                 radeon_program_register_sequence(rdev,
1070                                                  caicos_golden_registers,
1071                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1072                 break;
1073         default:
1074                 break;
1075         }
1076 }
1077
1078 /**
1079  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080  *
1081  * @rdev: radeon_device pointer
1082  * @reg: register offset in bytes
1083  * @val: register value
1084  *
1085  * Returns 0 for success or -EINVAL for an invalid register
1086  *
1087  */
1088 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089                                         u32 reg, u32 *val)
1090 {
1091         switch (reg) {
1092         case GRBM_STATUS:
1093         case GRBM_STATUS_SE0:
1094         case GRBM_STATUS_SE1:
1095         case SRBM_STATUS:
1096         case SRBM_STATUS2:
1097         case DMA_STATUS_REG:
1098         case UVD_STATUS:
1099                 *val = RREG32(reg);
1100                 return 0;
1101         default:
1102                 return -EINVAL;
1103         }
1104 }
1105
1106 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107                              unsigned *bankh, unsigned *mtaspect,
1108                              unsigned *tile_split)
1109 {
1110         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114         switch (*bankw) {
1115         default:
1116         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120         }
1121         switch (*bankh) {
1122         default:
1123         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127         }
1128         switch (*mtaspect) {
1129         default:
1130         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134         }
1135 }
1136
1137 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138                               u32 cntl_reg, u32 status_reg)
1139 {
1140         int r, i;
1141         struct atom_clock_dividers dividers;
1142
1143         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144                                            clock, false, &dividers);
1145         if (r)
1146                 return r;
1147
1148         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150         for (i = 0; i < 100; i++) {
1151                 if (RREG32(status_reg) & DCLK_STATUS)
1152                         break;
1153                 mdelay(10);
1154         }
1155         if (i == 100)
1156                 return -ETIMEDOUT;
1157
1158         return 0;
1159 }
1160
1161 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162 {
1163         int r = 0;
1164         u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167         if (r)
1168                 goto done;
1169         cg_scratch &= 0xffff0000;
1170         cg_scratch |= vclk / 100; /* Mhz */
1171
1172         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173         if (r)
1174                 goto done;
1175         cg_scratch &= 0x0000ffff;
1176         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178 done:
1179         WREG32(CG_SCRATCH1, cg_scratch);
1180
1181         return r;
1182 }
1183
1184 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185 {
1186         /* start off with something large */
1187         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1188         int r;
1189
1190         /* bypass vclk and dclk with bclk */
1191         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195         /* put PLL in bypass mode */
1196         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198         if (!vclk || !dclk) {
1199                 /* keep the Bypass mode, put PLL to sleep */
1200                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201                 return 0;
1202         }
1203
1204         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205                                           16384, 0x03FFFFFF, 0, 128, 5,
1206                                           &fb_div, &vclk_div, &dclk_div);
1207         if (r)
1208                 return r;
1209
1210         /* set VCO_MODE to 1 */
1211         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213         /* toggle UPLL_SLEEP to 1 then back to 0 */
1214         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217         /* deassert UPLL_RESET */
1218         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220         mdelay(1);
1221
1222         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1223         if (r)
1224                 return r;
1225
1226         /* assert UPLL_RESET again */
1227         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229         /* disable spread spectrum. */
1230         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232         /* set feedback divider */
1233         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1234
1235         /* set ref divider to 0 */
1236         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
1238         if (fb_div < 307200)
1239                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240         else
1241                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243         /* set PDIV_A and PDIV_B */
1244         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1245                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1246                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248         /* give the PLL some time to settle */
1249         mdelay(15);
1250
1251         /* deassert PLL_RESET */
1252         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254         mdelay(15);
1255
1256         /* switch from bypass mode to normal mode */
1257         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
1259         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1260         if (r)
1261                 return r;
1262
1263         /* switch VCLK and DCLK selection */
1264         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268         mdelay(100);
1269
1270         return 0;
1271 }
1272
1273 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274 {
1275         int readrq;
1276         u16 v;
1277
1278         readrq = pcie_get_readrq(rdev->pdev);
1279         v = ffs(readrq) - 8;
1280         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281          * to avoid hangs or perfomance issues
1282          */
1283         if ((v == 0) || (v == 6) || (v == 7))
1284                 pcie_set_readrq(rdev->pdev, 512);
1285 }
1286
1287 void dce4_program_fmt(struct drm_encoder *encoder)
1288 {
1289         struct drm_device *dev = encoder->dev;
1290         struct radeon_device *rdev = dev->dev_private;
1291         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294         int bpc = 0;
1295         u32 tmp = 0;
1296         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1297
1298         if (connector) {
1299                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1300                 bpc = radeon_get_monitor_bpc(connector);
1301                 dither = radeon_connector->dither;
1302         }
1303
1304         /* LVDS/eDP FMT is set up by atom */
1305         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306                 return;
1307
1308         /* not needed for analog */
1309         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311                 return;
1312
1313         if (bpc == 0)
1314                 return;
1315
1316         switch (bpc) {
1317         case 6:
1318                 if (dither == RADEON_FMT_DITHER_ENABLE)
1319                         /* XXX sort out optimal dither settings */
1320                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321                                 FMT_SPATIAL_DITHER_EN);
1322                 else
1323                         tmp |= FMT_TRUNCATE_EN;
1324                 break;
1325         case 8:
1326                 if (dither == RADEON_FMT_DITHER_ENABLE)
1327                         /* XXX sort out optimal dither settings */
1328                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329                                 FMT_RGB_RANDOM_ENABLE |
1330                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331                 else
1332                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333                 break;
1334         case 10:
1335         default:
1336                 /* not needed */
1337                 break;
1338         }
1339
1340         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341 }
1342
1343 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344 {
1345         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346                 return true;
1347         else
1348                 return false;
1349 }
1350
1351 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352 {
1353         u32 pos1, pos2;
1354
1355         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358         if (pos1 != pos2)
1359                 return true;
1360         else
1361                 return false;
1362 }
1363
1364 /**
1365  * dce4_wait_for_vblank - vblank wait asic callback.
1366  *
1367  * @rdev: radeon_device pointer
1368  * @crtc: crtc to wait for vblank on
1369  *
1370  * Wait for vblank on the requested crtc (evergreen+).
1371  */
1372 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373 {
1374         unsigned i = 0;
1375
1376         if (crtc >= rdev->num_crtc)
1377                 return;
1378
1379         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380                 return;
1381
1382         /* depending on when we hit vblank, we may be close to active; if so,
1383          * wait for another frame.
1384          */
1385         while (dce4_is_in_vblank(rdev, crtc)) {
1386                 if (i++ % 100 == 0) {
1387                         if (!dce4_is_counter_moving(rdev, crtc))
1388                                 break;
1389                 }
1390         }
1391
1392         while (!dce4_is_in_vblank(rdev, crtc)) {
1393                 if (i++ % 100 == 0) {
1394                         if (!dce4_is_counter_moving(rdev, crtc))
1395                                 break;
1396                 }
1397         }
1398 }
1399
1400 /**
1401  * evergreen_page_flip - pageflip callback.
1402  *
1403  * @rdev: radeon_device pointer
1404  * @crtc_id: crtc to cleanup pageflip on
1405  * @crtc_base: new address of the crtc (GPU MC address)
1406  *
1407  * Triggers the actual pageflip by updating the primary
1408  * surface base address (evergreen+).
1409  */
1410 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1411 {
1412         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1413
1414         /* update the scanout addresses */
1415         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1416                upper_32_bits(crtc_base));
1417         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1418                (u32)crtc_base);
1419         /* post the write */
1420         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1421 }
1422
1423 /**
1424  * evergreen_page_flip_pending - check if page flip is still pending
1425  *
1426  * @rdev: radeon_device pointer
1427  * @crtc_id: crtc to check
1428  *
1429  * Returns the current update pending status.
1430  */
1431 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1432 {
1433         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1434
1435         /* Return current update_pending status: */
1436         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1437                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1438 }
1439
1440 /* get temperature in millidegrees */
1441 int evergreen_get_temp(struct radeon_device *rdev)
1442 {
1443         u32 temp, toffset;
1444         int actual_temp = 0;
1445
1446         if (rdev->family == CHIP_JUNIPER) {
1447                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1448                         TOFFSET_SHIFT;
1449                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1450                         TS0_ADC_DOUT_SHIFT;
1451
1452                 if (toffset & 0x100)
1453                         actual_temp = temp / 2 - (0x200 - toffset);
1454                 else
1455                         actual_temp = temp / 2 + toffset;
1456
1457                 actual_temp = actual_temp * 1000;
1458
1459         } else {
1460                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1461                         ASIC_T_SHIFT;
1462
1463                 if (temp & 0x400)
1464                         actual_temp = -256;
1465                 else if (temp & 0x200)
1466                         actual_temp = 255;
1467                 else if (temp & 0x100) {
1468                         actual_temp = temp & 0x1ff;
1469                         actual_temp |= ~0x1ff;
1470                 } else
1471                         actual_temp = temp & 0xff;
1472
1473                 actual_temp = (actual_temp * 1000) / 2;
1474         }
1475
1476         return actual_temp;
1477 }
1478
1479 int sumo_get_temp(struct radeon_device *rdev)
1480 {
1481         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1482         int actual_temp = temp - 49;
1483
1484         return actual_temp * 1000;
1485 }
1486
1487 /**
1488  * sumo_pm_init_profile - Initialize power profiles callback.
1489  *
1490  * @rdev: radeon_device pointer
1491  *
1492  * Initialize the power states used in profile mode
1493  * (sumo, trinity, SI).
1494  * Used for profile mode only.
1495  */
1496 void sumo_pm_init_profile(struct radeon_device *rdev)
1497 {
1498         int idx;
1499
1500         /* default */
1501         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1502         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1503         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1504         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1505
1506         /* low,mid sh/mh */
1507         if (rdev->flags & RADEON_IS_MOBILITY)
1508                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1509         else
1510                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1511
1512         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1513         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1514         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1515         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1516
1517         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1518         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1519         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1520         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521
1522         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1526
1527         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1528         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1529         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1530         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1531
1532         /* high sh/mh */
1533         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1534         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1536         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1537         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1538                 rdev->pm.power_state[idx].num_clock_modes - 1;
1539
1540         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1541         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1542         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1543         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1544                 rdev->pm.power_state[idx].num_clock_modes - 1;
1545 }
1546
1547 /**
1548  * btc_pm_init_profile - Initialize power profiles callback.
1549  *
1550  * @rdev: radeon_device pointer
1551  *
1552  * Initialize the power states used in profile mode
1553  * (BTC, cayman).
1554  * Used for profile mode only.
1555  */
1556 void btc_pm_init_profile(struct radeon_device *rdev)
1557 {
1558         int idx;
1559
1560         /* default */
1561         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1562         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1563         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1564         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1565         /* starting with BTC, there is one state that is used for both
1566          * MH and SH.  Difference is that we always use the high clock index for
1567          * mclk.
1568          */
1569         if (rdev->flags & RADEON_IS_MOBILITY)
1570                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1571         else
1572                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1573         /* low sh */
1574         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1575         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1576         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1577         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1578         /* mid sh */
1579         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1580         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1581         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1582         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1583         /* high sh */
1584         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1585         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1586         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1587         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1588         /* low mh */
1589         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1590         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1591         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1592         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1593         /* mid mh */
1594         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1595         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1596         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1597         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1598         /* high mh */
1599         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1600         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1601         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1602         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1603 }
1604
1605 /**
1606  * evergreen_pm_misc - set additional pm hw parameters callback.
1607  *
1608  * @rdev: radeon_device pointer
1609  *
1610  * Set non-clock parameters associated with a power state
1611  * (voltage, etc.) (evergreen+).
1612  */
1613 void evergreen_pm_misc(struct radeon_device *rdev)
1614 {
1615         int req_ps_idx = rdev->pm.requested_power_state_index;
1616         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1617         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1618         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1619
1620         if (voltage->type == VOLTAGE_SW) {
1621                 /* 0xff0x are flags rather then an actual voltage */
1622                 if ((voltage->voltage & 0xff00) == 0xff00)
1623                         return;
1624                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1625                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1626                         rdev->pm.current_vddc = voltage->voltage;
1627                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1628                 }
1629
1630                 /* starting with BTC, there is one state that is used for both
1631                  * MH and SH.  Difference is that we always use the high clock index for
1632                  * mclk and vddci.
1633                  */
1634                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1635                     (rdev->family >= CHIP_BARTS) &&
1636                     rdev->pm.active_crtc_count &&
1637                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1638                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1639                         voltage = &rdev->pm.power_state[req_ps_idx].
1640                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1641
1642                 /* 0xff0x are flags rather then an actual voltage */
1643                 if ((voltage->vddci & 0xff00) == 0xff00)
1644                         return;
1645                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1646                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1647                         rdev->pm.current_vddci = voltage->vddci;
1648                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1649                 }
1650         }
1651 }
1652
1653 /**
1654  * evergreen_pm_prepare - pre-power state change callback.
1655  *
1656  * @rdev: radeon_device pointer
1657  *
1658  * Prepare for a power state change (evergreen+).
1659  */
1660 void evergreen_pm_prepare(struct radeon_device *rdev)
1661 {
1662         struct drm_device *ddev = rdev->ddev;
1663         struct drm_crtc *crtc;
1664         struct radeon_crtc *radeon_crtc;
1665         u32 tmp;
1666
1667         /* disable any active CRTCs */
1668         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1669                 radeon_crtc = to_radeon_crtc(crtc);
1670                 if (radeon_crtc->enabled) {
1671                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1672                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1673                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1674                 }
1675         }
1676 }
1677
1678 /**
1679  * evergreen_pm_finish - post-power state change callback.
1680  *
1681  * @rdev: radeon_device pointer
1682  *
1683  * Clean up after a power state change (evergreen+).
1684  */
1685 void evergreen_pm_finish(struct radeon_device *rdev)
1686 {
1687         struct drm_device *ddev = rdev->ddev;
1688         struct drm_crtc *crtc;
1689         struct radeon_crtc *radeon_crtc;
1690         u32 tmp;
1691
1692         /* enable any active CRTCs */
1693         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1694                 radeon_crtc = to_radeon_crtc(crtc);
1695                 if (radeon_crtc->enabled) {
1696                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1697                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1698                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1699                 }
1700         }
1701 }
1702
1703 /**
1704  * evergreen_hpd_sense - hpd sense callback.
1705  *
1706  * @rdev: radeon_device pointer
1707  * @hpd: hpd (hotplug detect) pin
1708  *
1709  * Checks if a digital monitor is connected (evergreen+).
1710  * Returns true if connected, false if not connected.
1711  */
1712 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1713 {
1714         bool connected = false;
1715
1716         switch (hpd) {
1717         case RADEON_HPD_1:
1718                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1719                         connected = true;
1720                 break;
1721         case RADEON_HPD_2:
1722                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1723                         connected = true;
1724                 break;
1725         case RADEON_HPD_3:
1726                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1727                         connected = true;
1728                 break;
1729         case RADEON_HPD_4:
1730                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1731                         connected = true;
1732                 break;
1733         case RADEON_HPD_5:
1734                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1735                         connected = true;
1736                 break;
1737         case RADEON_HPD_6:
1738                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1739                         connected = true;
1740                 break;
1741         default:
1742                 break;
1743         }
1744
1745         return connected;
1746 }
1747
1748 /**
1749  * evergreen_hpd_set_polarity - hpd set polarity callback.
1750  *
1751  * @rdev: radeon_device pointer
1752  * @hpd: hpd (hotplug detect) pin
1753  *
1754  * Set the polarity of the hpd pin (evergreen+).
1755  */
1756 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1757                                 enum radeon_hpd_id hpd)
1758 {
1759         u32 tmp;
1760         bool connected = evergreen_hpd_sense(rdev, hpd);
1761
1762         switch (hpd) {
1763         case RADEON_HPD_1:
1764                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1765                 if (connected)
1766                         tmp &= ~DC_HPDx_INT_POLARITY;
1767                 else
1768                         tmp |= DC_HPDx_INT_POLARITY;
1769                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1770                 break;
1771         case RADEON_HPD_2:
1772                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1773                 if (connected)
1774                         tmp &= ~DC_HPDx_INT_POLARITY;
1775                 else
1776                         tmp |= DC_HPDx_INT_POLARITY;
1777                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1778                 break;
1779         case RADEON_HPD_3:
1780                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1781                 if (connected)
1782                         tmp &= ~DC_HPDx_INT_POLARITY;
1783                 else
1784                         tmp |= DC_HPDx_INT_POLARITY;
1785                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1786                 break;
1787         case RADEON_HPD_4:
1788                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1789                 if (connected)
1790                         tmp &= ~DC_HPDx_INT_POLARITY;
1791                 else
1792                         tmp |= DC_HPDx_INT_POLARITY;
1793                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1794                 break;
1795         case RADEON_HPD_5:
1796                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1797                 if (connected)
1798                         tmp &= ~DC_HPDx_INT_POLARITY;
1799                 else
1800                         tmp |= DC_HPDx_INT_POLARITY;
1801                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1802                         break;
1803         case RADEON_HPD_6:
1804                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1805                 if (connected)
1806                         tmp &= ~DC_HPDx_INT_POLARITY;
1807                 else
1808                         tmp |= DC_HPDx_INT_POLARITY;
1809                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1810                 break;
1811         default:
1812                 break;
1813         }
1814 }
1815
1816 /**
1817  * evergreen_hpd_init - hpd setup callback.
1818  *
1819  * @rdev: radeon_device pointer
1820  *
1821  * Setup the hpd pins used by the card (evergreen+).
1822  * Enable the pin, set the polarity, and enable the hpd interrupts.
1823  */
1824 void evergreen_hpd_init(struct radeon_device *rdev)
1825 {
1826         struct drm_device *dev = rdev->ddev;
1827         struct drm_connector *connector;
1828         unsigned enabled = 0;
1829         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1830                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1831
1832         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1834
1835                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1836                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1837                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1838                          * aux dp channel on imac and help (but not completely fix)
1839                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1840                          * also avoid interrupt storms during dpms.
1841                          */
1842                         continue;
1843                 }
1844                 switch (radeon_connector->hpd.hpd) {
1845                 case RADEON_HPD_1:
1846                         WREG32(DC_HPD1_CONTROL, tmp);
1847                         break;
1848                 case RADEON_HPD_2:
1849                         WREG32(DC_HPD2_CONTROL, tmp);
1850                         break;
1851                 case RADEON_HPD_3:
1852                         WREG32(DC_HPD3_CONTROL, tmp);
1853                         break;
1854                 case RADEON_HPD_4:
1855                         WREG32(DC_HPD4_CONTROL, tmp);
1856                         break;
1857                 case RADEON_HPD_5:
1858                         WREG32(DC_HPD5_CONTROL, tmp);
1859                         break;
1860                 case RADEON_HPD_6:
1861                         WREG32(DC_HPD6_CONTROL, tmp);
1862                         break;
1863                 default:
1864                         break;
1865                 }
1866                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1867                 enabled |= 1 << radeon_connector->hpd.hpd;
1868         }
1869         radeon_irq_kms_enable_hpd(rdev, enabled);
1870 }
1871
1872 /**
1873  * evergreen_hpd_fini - hpd tear down callback.
1874  *
1875  * @rdev: radeon_device pointer
1876  *
1877  * Tear down the hpd pins used by the card (evergreen+).
1878  * Disable the hpd interrupts.
1879  */
1880 void evergreen_hpd_fini(struct radeon_device *rdev)
1881 {
1882         struct drm_device *dev = rdev->ddev;
1883         struct drm_connector *connector;
1884         unsigned disabled = 0;
1885
1886         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1887                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1888                 switch (radeon_connector->hpd.hpd) {
1889                 case RADEON_HPD_1:
1890                         WREG32(DC_HPD1_CONTROL, 0);
1891                         break;
1892                 case RADEON_HPD_2:
1893                         WREG32(DC_HPD2_CONTROL, 0);
1894                         break;
1895                 case RADEON_HPD_3:
1896                         WREG32(DC_HPD3_CONTROL, 0);
1897                         break;
1898                 case RADEON_HPD_4:
1899                         WREG32(DC_HPD4_CONTROL, 0);
1900                         break;
1901                 case RADEON_HPD_5:
1902                         WREG32(DC_HPD5_CONTROL, 0);
1903                         break;
1904                 case RADEON_HPD_6:
1905                         WREG32(DC_HPD6_CONTROL, 0);
1906                         break;
1907                 default:
1908                         break;
1909                 }
1910                 disabled |= 1 << radeon_connector->hpd.hpd;
1911         }
1912         radeon_irq_kms_disable_hpd(rdev, disabled);
1913 }
1914
1915 /* watermark setup */
1916
1917 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1918                                         struct radeon_crtc *radeon_crtc,
1919                                         struct drm_display_mode *mode,
1920                                         struct drm_display_mode *other_mode)
1921 {
1922         u32 tmp, buffer_alloc, i;
1923         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1924         /*
1925          * Line Buffer Setup
1926          * There are 3 line buffers, each one shared by 2 display controllers.
1927          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1928          * the display controllers.  The paritioning is done via one of four
1929          * preset allocations specified in bits 2:0:
1930          * first display controller
1931          *  0 - first half of lb (3840 * 2)
1932          *  1 - first 3/4 of lb (5760 * 2)
1933          *  2 - whole lb (7680 * 2), other crtc must be disabled
1934          *  3 - first 1/4 of lb (1920 * 2)
1935          * second display controller
1936          *  4 - second half of lb (3840 * 2)
1937          *  5 - second 3/4 of lb (5760 * 2)
1938          *  6 - whole lb (7680 * 2), other crtc must be disabled
1939          *  7 - last 1/4 of lb (1920 * 2)
1940          */
1941         /* this can get tricky if we have two large displays on a paired group
1942          * of crtcs.  Ideally for multiple large displays we'd assign them to
1943          * non-linked crtcs for maximum line buffer allocation.
1944          */
1945         if (radeon_crtc->base.enabled && mode) {
1946                 if (other_mode) {
1947                         tmp = 0; /* 1/2 */
1948                         buffer_alloc = 1;
1949                 } else {
1950                         tmp = 2; /* whole */
1951                         buffer_alloc = 2;
1952                 }
1953         } else {
1954                 tmp = 0;
1955                 buffer_alloc = 0;
1956         }
1957
1958         /* second controller of the pair uses second half of the lb */
1959         if (radeon_crtc->crtc_id % 2)
1960                 tmp += 4;
1961         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1962
1963         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1964                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1965                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1966                 for (i = 0; i < rdev->usec_timeout; i++) {
1967                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1968                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1969                                 break;
1970                         udelay(1);
1971                 }
1972         }
1973
1974         if (radeon_crtc->base.enabled && mode) {
1975                 switch (tmp) {
1976                 case 0:
1977                 case 4:
1978                 default:
1979                         if (ASIC_IS_DCE5(rdev))
1980                                 return 4096 * 2;
1981                         else
1982                                 return 3840 * 2;
1983                 case 1:
1984                 case 5:
1985                         if (ASIC_IS_DCE5(rdev))
1986                                 return 6144 * 2;
1987                         else
1988                                 return 5760 * 2;
1989                 case 2:
1990                 case 6:
1991                         if (ASIC_IS_DCE5(rdev))
1992                                 return 8192 * 2;
1993                         else
1994                                 return 7680 * 2;
1995                 case 3:
1996                 case 7:
1997                         if (ASIC_IS_DCE5(rdev))
1998                                 return 2048 * 2;
1999                         else
2000                                 return 1920 * 2;
2001                 }
2002         }
2003
2004         /* controller not enabled, so no lb used */
2005         return 0;
2006 }
2007
2008 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
2009 {
2010         u32 tmp = RREG32(MC_SHARED_CHMAP);
2011
2012         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2013         case 0:
2014         default:
2015                 return 1;
2016         case 1:
2017                 return 2;
2018         case 2:
2019                 return 4;
2020         case 3:
2021                 return 8;
2022         }
2023 }
2024
2025 struct evergreen_wm_params {
2026         u32 dram_channels; /* number of dram channels */
2027         u32 yclk;          /* bandwidth per dram data pin in kHz */
2028         u32 sclk;          /* engine clock in kHz */
2029         u32 disp_clk;      /* display clock in kHz */
2030         u32 src_width;     /* viewport width */
2031         u32 active_time;   /* active display time in ns */
2032         u32 blank_time;    /* blank time in ns */
2033         bool interlaced;    /* mode is interlaced */
2034         fixed20_12 vsc;    /* vertical scale ratio */
2035         u32 num_heads;     /* number of active crtcs */
2036         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2037         u32 lb_size;       /* line buffer allocated to pipe */
2038         u32 vtaps;         /* vertical scaler taps */
2039 };
2040
2041 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2042 {
2043         /* Calculate DRAM Bandwidth and the part allocated to display. */
2044         fixed20_12 dram_efficiency; /* 0.7 */
2045         fixed20_12 yclk, dram_channels, bandwidth;
2046         fixed20_12 a;
2047
2048         a.full = dfixed_const(1000);
2049         yclk.full = dfixed_const(wm->yclk);
2050         yclk.full = dfixed_div(yclk, a);
2051         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2052         a.full = dfixed_const(10);
2053         dram_efficiency.full = dfixed_const(7);
2054         dram_efficiency.full = dfixed_div(dram_efficiency, a);
2055         bandwidth.full = dfixed_mul(dram_channels, yclk);
2056         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2057
2058         return dfixed_trunc(bandwidth);
2059 }
2060
2061 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2062 {
2063         /* Calculate DRAM Bandwidth and the part allocated to display. */
2064         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2065         fixed20_12 yclk, dram_channels, bandwidth;
2066         fixed20_12 a;
2067
2068         a.full = dfixed_const(1000);
2069         yclk.full = dfixed_const(wm->yclk);
2070         yclk.full = dfixed_div(yclk, a);
2071         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2072         a.full = dfixed_const(10);
2073         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2074         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2075         bandwidth.full = dfixed_mul(dram_channels, yclk);
2076         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2077
2078         return dfixed_trunc(bandwidth);
2079 }
2080
2081 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2082 {
2083         /* Calculate the display Data return Bandwidth */
2084         fixed20_12 return_efficiency; /* 0.8 */
2085         fixed20_12 sclk, bandwidth;
2086         fixed20_12 a;
2087
2088         a.full = dfixed_const(1000);
2089         sclk.full = dfixed_const(wm->sclk);
2090         sclk.full = dfixed_div(sclk, a);
2091         a.full = dfixed_const(10);
2092         return_efficiency.full = dfixed_const(8);
2093         return_efficiency.full = dfixed_div(return_efficiency, a);
2094         a.full = dfixed_const(32);
2095         bandwidth.full = dfixed_mul(a, sclk);
2096         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2097
2098         return dfixed_trunc(bandwidth);
2099 }
2100
2101 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2102 {
2103         /* Calculate the DMIF Request Bandwidth */
2104         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2105         fixed20_12 disp_clk, bandwidth;
2106         fixed20_12 a;
2107
2108         a.full = dfixed_const(1000);
2109         disp_clk.full = dfixed_const(wm->disp_clk);
2110         disp_clk.full = dfixed_div(disp_clk, a);
2111         a.full = dfixed_const(10);
2112         disp_clk_request_efficiency.full = dfixed_const(8);
2113         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2114         a.full = dfixed_const(32);
2115         bandwidth.full = dfixed_mul(a, disp_clk);
2116         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2117
2118         return dfixed_trunc(bandwidth);
2119 }
2120
2121 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2122 {
2123         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2124         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2125         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2126         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2127
2128         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2129 }
2130
2131 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2132 {
2133         /* Calculate the display mode Average Bandwidth
2134          * DisplayMode should contain the source and destination dimensions,
2135          * timing, etc.
2136          */
2137         fixed20_12 bpp;
2138         fixed20_12 line_time;
2139         fixed20_12 src_width;
2140         fixed20_12 bandwidth;
2141         fixed20_12 a;
2142
2143         a.full = dfixed_const(1000);
2144         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2145         line_time.full = dfixed_div(line_time, a);
2146         bpp.full = dfixed_const(wm->bytes_per_pixel);
2147         src_width.full = dfixed_const(wm->src_width);
2148         bandwidth.full = dfixed_mul(src_width, bpp);
2149         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2150         bandwidth.full = dfixed_div(bandwidth, line_time);
2151
2152         return dfixed_trunc(bandwidth);
2153 }
2154
2155 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2156 {
2157         /* First calcualte the latency in ns */
2158         u32 mc_latency = 2000; /* 2000 ns. */
2159         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2160         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2161         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2162         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2163         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2164                 (wm->num_heads * cursor_line_pair_return_time);
2165         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2166         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2167         fixed20_12 a, b, c;
2168
2169         if (wm->num_heads == 0)
2170                 return 0;
2171
2172         a.full = dfixed_const(2);
2173         b.full = dfixed_const(1);
2174         if ((wm->vsc.full > a.full) ||
2175             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2176             (wm->vtaps >= 5) ||
2177             ((wm->vsc.full >= a.full) && wm->interlaced))
2178                 max_src_lines_per_dst_line = 4;
2179         else
2180                 max_src_lines_per_dst_line = 2;
2181
2182         a.full = dfixed_const(available_bandwidth);
2183         b.full = dfixed_const(wm->num_heads);
2184         a.full = dfixed_div(a, b);
2185
2186         b.full = dfixed_const(1000);
2187         c.full = dfixed_const(wm->disp_clk);
2188         b.full = dfixed_div(c, b);
2189         c.full = dfixed_const(wm->bytes_per_pixel);
2190         b.full = dfixed_mul(b, c);
2191
2192         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2193
2194         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2195         b.full = dfixed_const(1000);
2196         c.full = dfixed_const(lb_fill_bw);
2197         b.full = dfixed_div(c, b);
2198         a.full = dfixed_div(a, b);
2199         line_fill_time = dfixed_trunc(a);
2200
2201         if (line_fill_time < wm->active_time)
2202                 return latency;
2203         else
2204                 return latency + (line_fill_time - wm->active_time);
2205
2206 }
2207
2208 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2209 {
2210         if (evergreen_average_bandwidth(wm) <=
2211             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2212                 return true;
2213         else
2214                 return false;
2215 };
2216
2217 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2218 {
2219         if (evergreen_average_bandwidth(wm) <=
2220             (evergreen_available_bandwidth(wm) / wm->num_heads))
2221                 return true;
2222         else
2223                 return false;
2224 };
2225
2226 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2227 {
2228         u32 lb_partitions = wm->lb_size / wm->src_width;
2229         u32 line_time = wm->active_time + wm->blank_time;
2230         u32 latency_tolerant_lines;
2231         u32 latency_hiding;
2232         fixed20_12 a;
2233
2234         a.full = dfixed_const(1);
2235         if (wm->vsc.full > a.full)
2236                 latency_tolerant_lines = 1;
2237         else {
2238                 if (lb_partitions <= (wm->vtaps + 1))
2239                         latency_tolerant_lines = 1;
2240                 else
2241                         latency_tolerant_lines = 2;
2242         }
2243
2244         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2245
2246         if (evergreen_latency_watermark(wm) <= latency_hiding)
2247                 return true;
2248         else
2249                 return false;
2250 }
2251
2252 static void evergreen_program_watermarks(struct radeon_device *rdev,
2253                                          struct radeon_crtc *radeon_crtc,
2254                                          u32 lb_size, u32 num_heads)
2255 {
2256         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2257         struct evergreen_wm_params wm_low, wm_high;
2258         u32 dram_channels;
2259         u32 pixel_period;
2260         u32 line_time = 0;
2261         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2262         u32 priority_a_mark = 0, priority_b_mark = 0;
2263         u32 priority_a_cnt = PRIORITY_OFF;
2264         u32 priority_b_cnt = PRIORITY_OFF;
2265         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2266         u32 tmp, arb_control3;
2267         fixed20_12 a, b, c;
2268
2269         if (radeon_crtc->base.enabled && num_heads && mode) {
2270                 pixel_period = 1000000 / (u32)mode->clock;
2271                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2272                 priority_a_cnt = 0;
2273                 priority_b_cnt = 0;
2274                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275
2276                 /* watermark for high clocks */
2277                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2278                         wm_high.yclk =
2279                                 radeon_dpm_get_mclk(rdev, false) * 10;
2280                         wm_high.sclk =
2281                                 radeon_dpm_get_sclk(rdev, false) * 10;
2282                 } else {
2283                         wm_high.yclk = rdev->pm.current_mclk * 10;
2284                         wm_high.sclk = rdev->pm.current_sclk * 10;
2285                 }
2286
2287                 wm_high.disp_clk = mode->clock;
2288                 wm_high.src_width = mode->crtc_hdisplay;
2289                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2290                 wm_high.blank_time = line_time - wm_high.active_time;
2291                 wm_high.interlaced = false;
2292                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2293                         wm_high.interlaced = true;
2294                 wm_high.vsc = radeon_crtc->vsc;
2295                 wm_high.vtaps = 1;
2296                 if (radeon_crtc->rmx_type != RMX_OFF)
2297                         wm_high.vtaps = 2;
2298                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2299                 wm_high.lb_size = lb_size;
2300                 wm_high.dram_channels = dram_channels;
2301                 wm_high.num_heads = num_heads;
2302
2303                 /* watermark for low clocks */
2304                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2305                         wm_low.yclk =
2306                                 radeon_dpm_get_mclk(rdev, true) * 10;
2307                         wm_low.sclk =
2308                                 radeon_dpm_get_sclk(rdev, true) * 10;
2309                 } else {
2310                         wm_low.yclk = rdev->pm.current_mclk * 10;
2311                         wm_low.sclk = rdev->pm.current_sclk * 10;
2312                 }
2313
2314                 wm_low.disp_clk = mode->clock;
2315                 wm_low.src_width = mode->crtc_hdisplay;
2316                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2317                 wm_low.blank_time = line_time - wm_low.active_time;
2318                 wm_low.interlaced = false;
2319                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2320                         wm_low.interlaced = true;
2321                 wm_low.vsc = radeon_crtc->vsc;
2322                 wm_low.vtaps = 1;
2323                 if (radeon_crtc->rmx_type != RMX_OFF)
2324                         wm_low.vtaps = 2;
2325                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2326                 wm_low.lb_size = lb_size;
2327                 wm_low.dram_channels = dram_channels;
2328                 wm_low.num_heads = num_heads;
2329
2330                 /* set for high clocks */
2331                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2332                 /* set for low clocks */
2333                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2334
2335                 /* possibly force display priority to high */
2336                 /* should really do this at mode validation time... */
2337                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2338                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2339                     !evergreen_check_latency_hiding(&wm_high) ||
2340                     (rdev->disp_priority == 2)) {
2341                         DRM_DEBUG_KMS("force priority a to high\n");
2342                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2343                 }
2344                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346                     !evergreen_check_latency_hiding(&wm_low) ||
2347                     (rdev->disp_priority == 2)) {
2348                         DRM_DEBUG_KMS("force priority b to high\n");
2349                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2350                 }
2351
2352                 a.full = dfixed_const(1000);
2353                 b.full = dfixed_const(mode->clock);
2354                 b.full = dfixed_div(b, a);
2355                 c.full = dfixed_const(latency_watermark_a);
2356                 c.full = dfixed_mul(c, b);
2357                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2358                 c.full = dfixed_div(c, a);
2359                 a.full = dfixed_const(16);
2360                 c.full = dfixed_div(c, a);
2361                 priority_a_mark = dfixed_trunc(c);
2362                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2363
2364                 a.full = dfixed_const(1000);
2365                 b.full = dfixed_const(mode->clock);
2366                 b.full = dfixed_div(b, a);
2367                 c.full = dfixed_const(latency_watermark_b);
2368                 c.full = dfixed_mul(c, b);
2369                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2370                 c.full = dfixed_div(c, a);
2371                 a.full = dfixed_const(16);
2372                 c.full = dfixed_div(c, a);
2373                 priority_b_mark = dfixed_trunc(c);
2374                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2375
2376                 /* Save number of lines the linebuffer leads before the scanout */
2377                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2378         }
2379
2380         /* select wm A */
2381         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2382         tmp = arb_control3;
2383         tmp &= ~LATENCY_WATERMARK_MASK(3);
2384         tmp |= LATENCY_WATERMARK_MASK(1);
2385         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2386         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2387                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2388                 LATENCY_HIGH_WATERMARK(line_time)));
2389         /* select wm B */
2390         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2391         tmp &= ~LATENCY_WATERMARK_MASK(3);
2392         tmp |= LATENCY_WATERMARK_MASK(2);
2393         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2394         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2395                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2396                 LATENCY_HIGH_WATERMARK(line_time)));
2397         /* restore original selection */
2398         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2399
2400         /* write the priority marks */
2401         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2402         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2403
2404         /* save values for DPM */
2405         radeon_crtc->line_time = line_time;
2406         radeon_crtc->wm_high = latency_watermark_a;
2407         radeon_crtc->wm_low = latency_watermark_b;
2408 }
2409
2410 /**
2411  * evergreen_bandwidth_update - update display watermarks callback.
2412  *
2413  * @rdev: radeon_device pointer
2414  *
2415  * Update the display watermarks based on the requested mode(s)
2416  * (evergreen+).
2417  */
2418 void evergreen_bandwidth_update(struct radeon_device *rdev)
2419 {
2420         struct drm_display_mode *mode0 = NULL;
2421         struct drm_display_mode *mode1 = NULL;
2422         u32 num_heads = 0, lb_size;
2423         int i;
2424
2425         if (!rdev->mode_info.mode_config_initialized)
2426                 return;
2427
2428         radeon_update_display_priority(rdev);
2429
2430         for (i = 0; i < rdev->num_crtc; i++) {
2431                 if (rdev->mode_info.crtcs[i]->base.enabled)
2432                         num_heads++;
2433         }
2434         for (i = 0; i < rdev->num_crtc; i += 2) {
2435                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2436                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2437                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2438                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2439                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2440                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2441         }
2442 }
2443
2444 /**
2445  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2446  *
2447  * @rdev: radeon_device pointer
2448  *
2449  * Wait for the MC (memory controller) to be idle.
2450  * (evergreen+).
2451  * Returns 0 if the MC is idle, -1 if not.
2452  */
2453 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2454 {
2455         unsigned i;
2456         u32 tmp;
2457
2458         for (i = 0; i < rdev->usec_timeout; i++) {
2459                 /* read MC_STATUS */
2460                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2461                 if (!tmp)
2462                         return 0;
2463                 udelay(1);
2464         }
2465         return -1;
2466 }
2467
2468 /*
2469  * GART
2470  */
2471 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2472 {
2473         unsigned i;
2474         u32 tmp;
2475
2476         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2477
2478         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2479         for (i = 0; i < rdev->usec_timeout; i++) {
2480                 /* read MC_STATUS */
2481                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2482                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2483                 if (tmp == 2) {
2484                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2485                         return;
2486                 }
2487                 if (tmp) {
2488                         return;
2489                 }
2490                 udelay(1);
2491         }
2492 }
2493
2494 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2495 {
2496         u32 tmp;
2497         int r;
2498
2499         if (rdev->gart.robj == NULL) {
2500                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2501                 return -EINVAL;
2502         }
2503         r = radeon_gart_table_vram_pin(rdev);
2504         if (r)
2505                 return r;
2506         /* Setup L2 cache */
2507         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2508                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2509                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2510         WREG32(VM_L2_CNTL2, 0);
2511         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2512         /* Setup TLB control */
2513         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2514                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2515                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2516                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2517         if (rdev->flags & RADEON_IS_IGP) {
2518                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2519                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2520                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2521         } else {
2522                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2523                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2524                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2525                 if ((rdev->family == CHIP_JUNIPER) ||
2526                     (rdev->family == CHIP_CYPRESS) ||
2527                     (rdev->family == CHIP_HEMLOCK) ||
2528                     (rdev->family == CHIP_BARTS))
2529                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2530         }
2531         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2532         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2533         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2534         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2535         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2536         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2537         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2538         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2539                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2540         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2541                         (u32)(rdev->dummy_page.addr >> 12));
2542         WREG32(VM_CONTEXT1_CNTL, 0);
2543
2544         evergreen_pcie_gart_tlb_flush(rdev);
2545         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2546                  (unsigned)(rdev->mc.gtt_size >> 20),
2547                  (unsigned long long)rdev->gart.table_addr);
2548         rdev->gart.ready = true;
2549         return 0;
2550 }
2551
2552 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2553 {
2554         u32 tmp;
2555
2556         /* Disable all tables */
2557         WREG32(VM_CONTEXT0_CNTL, 0);
2558         WREG32(VM_CONTEXT1_CNTL, 0);
2559
2560         /* Setup L2 cache */
2561         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2562                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2563         WREG32(VM_L2_CNTL2, 0);
2564         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2565         /* Setup TLB control */
2566         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2567         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2568         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2569         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2570         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2571         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2572         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2573         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2574         radeon_gart_table_vram_unpin(rdev);
2575 }
2576
2577 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2578 {
2579         evergreen_pcie_gart_disable(rdev);
2580         radeon_gart_table_vram_free(rdev);
2581         radeon_gart_fini(rdev);
2582 }
2583
2584
2585 static void evergreen_agp_enable(struct radeon_device *rdev)
2586 {
2587         u32 tmp;
2588
2589         /* Setup L2 cache */
2590         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2591                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2592                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2593         WREG32(VM_L2_CNTL2, 0);
2594         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2595         /* Setup TLB control */
2596         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2597                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2598                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2599                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2600         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2601         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2602         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2603         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2604         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2605         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2606         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2607         WREG32(VM_CONTEXT0_CNTL, 0);
2608         WREG32(VM_CONTEXT1_CNTL, 0);
2609 }
2610
2611 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2612 {
2613         u32 crtc_enabled, tmp, frame_count, blackout;
2614         int i, j;
2615
2616         if (!ASIC_IS_NODCE(rdev)) {
2617                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2618                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2619
2620                 /* disable VGA render */
2621                 WREG32(VGA_RENDER_CONTROL, 0);
2622         }
2623         /* blank the display controllers */
2624         for (i = 0; i < rdev->num_crtc; i++) {
2625                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2626                 if (crtc_enabled) {
2627                         save->crtc_enabled[i] = true;
2628                         if (ASIC_IS_DCE6(rdev)) {
2629                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2630                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2631                                         radeon_wait_for_vblank(rdev, i);
2632                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2633                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2634                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2635                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2636                                 }
2637                         } else {
2638                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2639                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2640                                         radeon_wait_for_vblank(rdev, i);
2641                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2642                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2643                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2644                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2645                                 }
2646                         }
2647                         /* wait for the next frame */
2648                         frame_count = radeon_get_vblank_counter(rdev, i);
2649                         for (j = 0; j < rdev->usec_timeout; j++) {
2650                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2651                                         break;
2652                                 udelay(1);
2653                         }
2654
2655                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2656                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2657                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2658                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2659                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2660                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2661                         save->crtc_enabled[i] = false;
2662                         /* ***** */
2663                 } else {
2664                         save->crtc_enabled[i] = false;
2665                 }
2666         }
2667
2668         radeon_mc_wait_for_idle(rdev);
2669
2670         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2671         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2672                 /* Block CPU access */
2673                 WREG32(BIF_FB_EN, 0);
2674                 /* blackout the MC */
2675                 blackout &= ~BLACKOUT_MODE_MASK;
2676                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2677         }
2678         /* wait for the MC to settle */
2679         udelay(100);
2680
2681         /* lock double buffered regs */
2682         for (i = 0; i < rdev->num_crtc; i++) {
2683                 if (save->crtc_enabled[i]) {
2684                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2685                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2686                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2687                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2688                         }
2689                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2690                         if (!(tmp & 1)) {
2691                                 tmp |= 1;
2692                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2693                         }
2694                 }
2695         }
2696 }
2697
2698 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2699 {
2700         u32 tmp, frame_count;
2701         int i, j;
2702
2703         /* update crtc base addresses */
2704         for (i = 0; i < rdev->num_crtc; i++) {
2705                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2706                        upper_32_bits(rdev->mc.vram_start));
2707                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2708                        upper_32_bits(rdev->mc.vram_start));
2709                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2710                        (u32)rdev->mc.vram_start);
2711                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2712                        (u32)rdev->mc.vram_start);
2713         }
2714
2715         if (!ASIC_IS_NODCE(rdev)) {
2716                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2717                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2718         }
2719
2720         /* unlock regs and wait for update */
2721         for (i = 0; i < rdev->num_crtc; i++) {
2722                 if (save->crtc_enabled[i]) {
2723                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2724                         if ((tmp & 0x7) != 3) {
2725                                 tmp &= ~0x7;
2726                                 tmp |= 0x3;
2727                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2728                         }
2729                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2730                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2731                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2732                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2733                         }
2734                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2735                         if (tmp & 1) {
2736                                 tmp &= ~1;
2737                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2738                         }
2739                         for (j = 0; j < rdev->usec_timeout; j++) {
2740                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2741                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2742                                         break;
2743                                 udelay(1);
2744                         }
2745                 }
2746         }
2747
2748         /* unblackout the MC */
2749         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2750         tmp &= ~BLACKOUT_MODE_MASK;
2751         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2752         /* allow CPU access */
2753         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2754
2755         for (i = 0; i < rdev->num_crtc; i++) {
2756                 if (save->crtc_enabled[i]) {
2757                         if (ASIC_IS_DCE6(rdev)) {
2758                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2759                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2760                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2761                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2762                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2763                         } else {
2764                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2765                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2766                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2767                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2768                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2769                         }
2770                         /* wait for the next frame */
2771                         frame_count = radeon_get_vblank_counter(rdev, i);
2772                         for (j = 0; j < rdev->usec_timeout; j++) {
2773                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2774                                         break;
2775                                 udelay(1);
2776                         }
2777                 }
2778         }
2779         if (!ASIC_IS_NODCE(rdev)) {
2780                 /* Unlock vga access */
2781                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2782                 mdelay(1);
2783                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2784         }
2785 }
2786
2787 void evergreen_mc_program(struct radeon_device *rdev)
2788 {
2789         struct evergreen_mc_save save;
2790         u32 tmp;
2791         int i, j;
2792
2793         /* Initialize HDP */
2794         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2795                 WREG32((0x2c14 + j), 0x00000000);
2796                 WREG32((0x2c18 + j), 0x00000000);
2797                 WREG32((0x2c1c + j), 0x00000000);
2798                 WREG32((0x2c20 + j), 0x00000000);
2799                 WREG32((0x2c24 + j), 0x00000000);
2800         }
2801         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2802
2803         evergreen_mc_stop(rdev, &save);
2804         if (evergreen_mc_wait_for_idle(rdev)) {
2805                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2806         }
2807         /* Lockout access through VGA aperture*/
2808         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2809         /* Update configuration */
2810         if (rdev->flags & RADEON_IS_AGP) {
2811                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2812                         /* VRAM before AGP */
2813                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2814                                 rdev->mc.vram_start >> 12);
2815                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2816                                 rdev->mc.gtt_end >> 12);
2817                 } else {
2818                         /* VRAM after AGP */
2819                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2820                                 rdev->mc.gtt_start >> 12);
2821                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2822                                 rdev->mc.vram_end >> 12);
2823                 }
2824         } else {
2825                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2826                         rdev->mc.vram_start >> 12);
2827                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2828                         rdev->mc.vram_end >> 12);
2829         }
2830         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2831         /* llano/ontario only */
2832         if ((rdev->family == CHIP_PALM) ||
2833             (rdev->family == CHIP_SUMO) ||
2834             (rdev->family == CHIP_SUMO2)) {
2835                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2836                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2837                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2838                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2839         }
2840         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2841         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2842         WREG32(MC_VM_FB_LOCATION, tmp);
2843         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2844         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2845         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2846         if (rdev->flags & RADEON_IS_AGP) {
2847                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2848                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2849                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2850         } else {
2851                 WREG32(MC_VM_AGP_BASE, 0);
2852                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2853                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2854         }
2855         if (evergreen_mc_wait_for_idle(rdev)) {
2856                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2857         }
2858         evergreen_mc_resume(rdev, &save);
2859         /* we need to own VRAM, so turn off the VGA renderer here
2860          * to stop it overwriting our objects */
2861         rv515_vga_render_disable(rdev);
2862 }
2863
2864 /*
2865  * CP.
2866  */
2867 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2868 {
2869         struct radeon_ring *ring = &rdev->ring[ib->ring];
2870         u32 next_rptr;
2871
2872         /* set to DX10/11 mode */
2873         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2874         radeon_ring_write(ring, 1);
2875
2876         if (ring->rptr_save_reg) {
2877                 next_rptr = ring->wptr + 3 + 4;
2878                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2879                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2880                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2881                 radeon_ring_write(ring, next_rptr);
2882         } else if (rdev->wb.enabled) {
2883                 next_rptr = ring->wptr + 5 + 4;
2884                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2885                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2886                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2887                 radeon_ring_write(ring, next_rptr);
2888                 radeon_ring_write(ring, 0);
2889         }
2890
2891         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2892         radeon_ring_write(ring,
2893 #ifdef __BIG_ENDIAN
2894                           (2 << 0) |
2895 #endif
2896                           (ib->gpu_addr & 0xFFFFFFFC));
2897         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2898         radeon_ring_write(ring, ib->length_dw);
2899 }
2900
2901
2902 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2903 {
2904         const __be32 *fw_data;
2905         int i;
2906
2907         if (!rdev->me_fw || !rdev->pfp_fw)
2908                 return -EINVAL;
2909
2910         r700_cp_stop(rdev);
2911         WREG32(CP_RB_CNTL,
2912 #ifdef __BIG_ENDIAN
2913                BUF_SWAP_32BIT |
2914 #endif
2915                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2916
2917         fw_data = (const __be32 *)rdev->pfp_fw->data;
2918         WREG32(CP_PFP_UCODE_ADDR, 0);
2919         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2920                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2921         WREG32(CP_PFP_UCODE_ADDR, 0);
2922
2923         fw_data = (const __be32 *)rdev->me_fw->data;
2924         WREG32(CP_ME_RAM_WADDR, 0);
2925         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2926                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2927
2928         WREG32(CP_PFP_UCODE_ADDR, 0);
2929         WREG32(CP_ME_RAM_WADDR, 0);
2930         WREG32(CP_ME_RAM_RADDR, 0);
2931         return 0;
2932 }
2933
2934 static int evergreen_cp_start(struct radeon_device *rdev)
2935 {
2936         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2937         int r, i;
2938         uint32_t cp_me;
2939
2940         r = radeon_ring_lock(rdev, ring, 7);
2941         if (r) {
2942                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2943                 return r;
2944         }
2945         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2946         radeon_ring_write(ring, 0x1);
2947         radeon_ring_write(ring, 0x0);
2948         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2949         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2950         radeon_ring_write(ring, 0);
2951         radeon_ring_write(ring, 0);
2952         radeon_ring_unlock_commit(rdev, ring, false);
2953
2954         cp_me = 0xff;
2955         WREG32(CP_ME_CNTL, cp_me);
2956
2957         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2958         if (r) {
2959                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2960                 return r;
2961         }
2962
2963         /* setup clear context state */
2964         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2965         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2966
2967         for (i = 0; i < evergreen_default_size; i++)
2968                 radeon_ring_write(ring, evergreen_default_state[i]);
2969
2970         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2971         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2972
2973         /* set clear context state */
2974         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2975         radeon_ring_write(ring, 0);
2976
2977         /* SQ_VTX_BASE_VTX_LOC */
2978         radeon_ring_write(ring, 0xc0026f00);
2979         radeon_ring_write(ring, 0x00000000);
2980         radeon_ring_write(ring, 0x00000000);
2981         radeon_ring_write(ring, 0x00000000);
2982
2983         /* Clear consts */
2984         radeon_ring_write(ring, 0xc0036f00);
2985         radeon_ring_write(ring, 0x00000bc4);
2986         radeon_ring_write(ring, 0xffffffff);
2987         radeon_ring_write(ring, 0xffffffff);
2988         radeon_ring_write(ring, 0xffffffff);
2989
2990         radeon_ring_write(ring, 0xc0026900);
2991         radeon_ring_write(ring, 0x00000316);
2992         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2993         radeon_ring_write(ring, 0x00000010); /*  */
2994
2995         radeon_ring_unlock_commit(rdev, ring, false);
2996
2997         return 0;
2998 }
2999
3000 static int evergreen_cp_resume(struct radeon_device *rdev)
3001 {
3002         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3003         u32 tmp;
3004         u32 rb_bufsz;
3005         int r;
3006
3007         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3008         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3009                                  SOFT_RESET_PA |
3010                                  SOFT_RESET_SH |
3011                                  SOFT_RESET_VGT |
3012                                  SOFT_RESET_SPI |
3013                                  SOFT_RESET_SX));
3014         RREG32(GRBM_SOFT_RESET);
3015         mdelay(15);
3016         WREG32(GRBM_SOFT_RESET, 0);
3017         RREG32(GRBM_SOFT_RESET);
3018
3019         /* Set ring buffer size */
3020         rb_bufsz = order_base_2(ring->ring_size / 8);
3021         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3022 #ifdef __BIG_ENDIAN
3023         tmp |= BUF_SWAP_32BIT;
3024 #endif
3025         WREG32(CP_RB_CNTL, tmp);
3026         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3027         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3028
3029         /* Set the write pointer delay */
3030         WREG32(CP_RB_WPTR_DELAY, 0);
3031
3032         /* Initialize the ring buffer's read and write pointers */
3033         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3034         WREG32(CP_RB_RPTR_WR, 0);
3035         ring->wptr = 0;
3036         WREG32(CP_RB_WPTR, ring->wptr);
3037
3038         /* set the wb address whether it's enabled or not */
3039         WREG32(CP_RB_RPTR_ADDR,
3040                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3041         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3042         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3043
3044         if (rdev->wb.enabled)
3045                 WREG32(SCRATCH_UMSK, 0xff);
3046         else {
3047                 tmp |= RB_NO_UPDATE;
3048                 WREG32(SCRATCH_UMSK, 0);
3049         }
3050
3051         mdelay(1);
3052         WREG32(CP_RB_CNTL, tmp);
3053
3054         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3055         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3056
3057         evergreen_cp_start(rdev);
3058         ring->ready = true;
3059         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3060         if (r) {
3061                 ring->ready = false;
3062                 return r;
3063         }
3064         return 0;
3065 }
3066
3067 /*
3068  * Core functions
3069  */
3070 static void evergreen_gpu_init(struct radeon_device *rdev)
3071 {
3072         u32 gb_addr_config;
3073         u32 mc_shared_chmap, mc_arb_ramcfg;
3074         u32 sx_debug_1;
3075         u32 smx_dc_ctl0;
3076         u32 sq_config;
3077         u32 sq_lds_resource_mgmt;
3078         u32 sq_gpr_resource_mgmt_1;
3079         u32 sq_gpr_resource_mgmt_2;
3080         u32 sq_gpr_resource_mgmt_3;
3081         u32 sq_thread_resource_mgmt;
3082         u32 sq_thread_resource_mgmt_2;
3083         u32 sq_stack_resource_mgmt_1;
3084         u32 sq_stack_resource_mgmt_2;
3085         u32 sq_stack_resource_mgmt_3;
3086         u32 vgt_cache_invalidation;
3087         u32 hdp_host_path_cntl, tmp;
3088         u32 disabled_rb_mask;
3089         int i, j, ps_thread_count;
3090
3091         switch (rdev->family) {
3092         case CHIP_CYPRESS:
3093         case CHIP_HEMLOCK:
3094                 rdev->config.evergreen.num_ses = 2;
3095                 rdev->config.evergreen.max_pipes = 4;
3096                 rdev->config.evergreen.max_tile_pipes = 8;
3097                 rdev->config.evergreen.max_simds = 10;
3098                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3099                 rdev->config.evergreen.max_gprs = 256;
3100                 rdev->config.evergreen.max_threads = 248;
3101                 rdev->config.evergreen.max_gs_threads = 32;
3102                 rdev->config.evergreen.max_stack_entries = 512;
3103                 rdev->config.evergreen.sx_num_of_sets = 4;
3104                 rdev->config.evergreen.sx_max_export_size = 256;
3105                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3106                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3107                 rdev->config.evergreen.max_hw_contexts = 8;
3108                 rdev->config.evergreen.sq_num_cf_insts = 2;
3109
3110                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3111                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3112                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3113                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3114                 break;
3115         case CHIP_JUNIPER:
3116                 rdev->config.evergreen.num_ses = 1;
3117                 rdev->config.evergreen.max_pipes = 4;
3118                 rdev->config.evergreen.max_tile_pipes = 4;
3119                 rdev->config.evergreen.max_simds = 10;
3120                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3121                 rdev->config.evergreen.max_gprs = 256;
3122                 rdev->config.evergreen.max_threads = 248;
3123                 rdev->config.evergreen.max_gs_threads = 32;
3124                 rdev->config.evergreen.max_stack_entries = 512;
3125                 rdev->config.evergreen.sx_num_of_sets = 4;
3126                 rdev->config.evergreen.sx_max_export_size = 256;
3127                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3128                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3129                 rdev->config.evergreen.max_hw_contexts = 8;
3130                 rdev->config.evergreen.sq_num_cf_insts = 2;
3131
3132                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3133                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3134                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3135                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3136                 break;
3137         case CHIP_REDWOOD:
3138                 rdev->config.evergreen.num_ses = 1;
3139                 rdev->config.evergreen.max_pipes = 4;
3140                 rdev->config.evergreen.max_tile_pipes = 4;
3141                 rdev->config.evergreen.max_simds = 5;
3142                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3143                 rdev->config.evergreen.max_gprs = 256;
3144                 rdev->config.evergreen.max_threads = 248;
3145                 rdev->config.evergreen.max_gs_threads = 32;
3146                 rdev->config.evergreen.max_stack_entries = 256;
3147                 rdev->config.evergreen.sx_num_of_sets = 4;
3148                 rdev->config.evergreen.sx_max_export_size = 256;
3149                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3150                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3151                 rdev->config.evergreen.max_hw_contexts = 8;
3152                 rdev->config.evergreen.sq_num_cf_insts = 2;
3153
3154                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3155                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3156                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3157                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3158                 break;
3159         case CHIP_CEDAR:
3160         default:
3161                 rdev->config.evergreen.num_ses = 1;
3162                 rdev->config.evergreen.max_pipes = 2;
3163                 rdev->config.evergreen.max_tile_pipes = 2;
3164                 rdev->config.evergreen.max_simds = 2;
3165                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3166                 rdev->config.evergreen.max_gprs = 256;
3167                 rdev->config.evergreen.max_threads = 192;
3168                 rdev->config.evergreen.max_gs_threads = 16;
3169                 rdev->config.evergreen.max_stack_entries = 256;
3170                 rdev->config.evergreen.sx_num_of_sets = 4;
3171                 rdev->config.evergreen.sx_max_export_size = 128;
3172                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3173                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3174                 rdev->config.evergreen.max_hw_contexts = 4;
3175                 rdev->config.evergreen.sq_num_cf_insts = 1;
3176
3177                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3178                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3179                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3180                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3181                 break;
3182         case CHIP_PALM:
3183                 rdev->config.evergreen.num_ses = 1;
3184                 rdev->config.evergreen.max_pipes = 2;
3185                 rdev->config.evergreen.max_tile_pipes = 2;
3186                 rdev->config.evergreen.max_simds = 2;
3187                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3188                 rdev->config.evergreen.max_gprs = 256;
3189                 rdev->config.evergreen.max_threads = 192;
3190                 rdev->config.evergreen.max_gs_threads = 16;
3191                 rdev->config.evergreen.max_stack_entries = 256;
3192                 rdev->config.evergreen.sx_num_of_sets = 4;
3193                 rdev->config.evergreen.sx_max_export_size = 128;
3194                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3195                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3196                 rdev->config.evergreen.max_hw_contexts = 4;
3197                 rdev->config.evergreen.sq_num_cf_insts = 1;
3198
3199                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3200                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3201                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3202                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3203                 break;
3204         case CHIP_SUMO:
3205                 rdev->config.evergreen.num_ses = 1;
3206                 rdev->config.evergreen.max_pipes = 4;
3207                 rdev->config.evergreen.max_tile_pipes = 4;
3208                 if (rdev->pdev->device == 0x9648)
3209                         rdev->config.evergreen.max_simds = 3;
3210                 else if ((rdev->pdev->device == 0x9647) ||
3211                          (rdev->pdev->device == 0x964a))
3212                         rdev->config.evergreen.max_simds = 4;
3213                 else
3214                         rdev->config.evergreen.max_simds = 5;
3215                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3216                 rdev->config.evergreen.max_gprs = 256;
3217                 rdev->config.evergreen.max_threads = 248;
3218                 rdev->config.evergreen.max_gs_threads = 32;
3219                 rdev->config.evergreen.max_stack_entries = 256;
3220                 rdev->config.evergreen.sx_num_of_sets = 4;
3221                 rdev->config.evergreen.sx_max_export_size = 256;
3222                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3223                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3224                 rdev->config.evergreen.max_hw_contexts = 8;
3225                 rdev->config.evergreen.sq_num_cf_insts = 2;
3226
3227                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3228                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3229                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3230                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3231                 break;
3232         case CHIP_SUMO2:
3233                 rdev->config.evergreen.num_ses = 1;
3234                 rdev->config.evergreen.max_pipes = 4;
3235                 rdev->config.evergreen.max_tile_pipes = 4;
3236                 rdev->config.evergreen.max_simds = 2;
3237                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3238                 rdev->config.evergreen.max_gprs = 256;
3239                 rdev->config.evergreen.max_threads = 248;
3240                 rdev->config.evergreen.max_gs_threads = 32;
3241                 rdev->config.evergreen.max_stack_entries = 512;
3242                 rdev->config.evergreen.sx_num_of_sets = 4;
3243                 rdev->config.evergreen.sx_max_export_size = 256;
3244                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3245                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3246                 rdev->config.evergreen.max_hw_contexts = 4;
3247                 rdev->config.evergreen.sq_num_cf_insts = 2;
3248
3249                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3250                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3251                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3252                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3253                 break;
3254         case CHIP_BARTS:
3255                 rdev->config.evergreen.num_ses = 2;
3256                 rdev->config.evergreen.max_pipes = 4;
3257                 rdev->config.evergreen.max_tile_pipes = 8;
3258                 rdev->config.evergreen.max_simds = 7;
3259                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3260                 rdev->config.evergreen.max_gprs = 256;
3261                 rdev->config.evergreen.max_threads = 248;
3262                 rdev->config.evergreen.max_gs_threads = 32;
3263                 rdev->config.evergreen.max_stack_entries = 512;
3264                 rdev->config.evergreen.sx_num_of_sets = 4;
3265                 rdev->config.evergreen.sx_max_export_size = 256;
3266                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3267                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3268                 rdev->config.evergreen.max_hw_contexts = 8;
3269                 rdev->config.evergreen.sq_num_cf_insts = 2;
3270
3271                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3272                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3273                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3274                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3275                 break;
3276         case CHIP_TURKS:
3277                 rdev->config.evergreen.num_ses = 1;
3278                 rdev->config.evergreen.max_pipes = 4;
3279                 rdev->config.evergreen.max_tile_pipes = 4;
3280                 rdev->config.evergreen.max_simds = 6;
3281                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3282                 rdev->config.evergreen.max_gprs = 256;
3283                 rdev->config.evergreen.max_threads = 248;
3284                 rdev->config.evergreen.max_gs_threads = 32;
3285                 rdev->config.evergreen.max_stack_entries = 256;
3286                 rdev->config.evergreen.sx_num_of_sets = 4;
3287                 rdev->config.evergreen.sx_max_export_size = 256;
3288                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3289                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3290                 rdev->config.evergreen.max_hw_contexts = 8;
3291                 rdev->config.evergreen.sq_num_cf_insts = 2;
3292
3293                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3294                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3295                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3296                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3297                 break;
3298         case CHIP_CAICOS:
3299                 rdev->config.evergreen.num_ses = 1;
3300                 rdev->config.evergreen.max_pipes = 2;
3301                 rdev->config.evergreen.max_tile_pipes = 2;
3302                 rdev->config.evergreen.max_simds = 2;
3303                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3304                 rdev->config.evergreen.max_gprs = 256;
3305                 rdev->config.evergreen.max_threads = 192;
3306                 rdev->config.evergreen.max_gs_threads = 16;
3307                 rdev->config.evergreen.max_stack_entries = 256;
3308                 rdev->config.evergreen.sx_num_of_sets = 4;
3309                 rdev->config.evergreen.sx_max_export_size = 128;
3310                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3311                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3312                 rdev->config.evergreen.max_hw_contexts = 4;
3313                 rdev->config.evergreen.sq_num_cf_insts = 1;
3314
3315                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3316                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3317                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3318                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3319                 break;
3320         }
3321
3322         /* Initialize HDP */
3323         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3324                 WREG32((0x2c14 + j), 0x00000000);
3325                 WREG32((0x2c18 + j), 0x00000000);
3326                 WREG32((0x2c1c + j), 0x00000000);
3327                 WREG32((0x2c20 + j), 0x00000000);
3328                 WREG32((0x2c24 + j), 0x00000000);
3329         }
3330
3331         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3332         WREG32(SRBM_INT_CNTL, 0x1);
3333         WREG32(SRBM_INT_ACK, 0x1);
3334
3335         evergreen_fix_pci_max_read_req_size(rdev);
3336
3337         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3338         if ((rdev->family == CHIP_PALM) ||
3339             (rdev->family == CHIP_SUMO) ||
3340             (rdev->family == CHIP_SUMO2))
3341                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3342         else
3343                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3344
3345         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3346          * not have bank info, so create a custom tiling dword.
3347          * bits 3:0   num_pipes
3348          * bits 7:4   num_banks
3349          * bits 11:8  group_size
3350          * bits 15:12 row_size
3351          */
3352         rdev->config.evergreen.tile_config = 0;
3353         switch (rdev->config.evergreen.max_tile_pipes) {
3354         case 1:
3355         default:
3356                 rdev->config.evergreen.tile_config |= (0 << 0);
3357                 break;
3358         case 2:
3359                 rdev->config.evergreen.tile_config |= (1 << 0);
3360                 break;
3361         case 4:
3362                 rdev->config.evergreen.tile_config |= (2 << 0);
3363                 break;
3364         case 8:
3365                 rdev->config.evergreen.tile_config |= (3 << 0);
3366                 break;
3367         }
3368         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3369         if (rdev->flags & RADEON_IS_IGP)
3370                 rdev->config.evergreen.tile_config |= 1 << 4;
3371         else {
3372                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3373                 case 0: /* four banks */
3374                         rdev->config.evergreen.tile_config |= 0 << 4;
3375                         break;
3376                 case 1: /* eight banks */
3377                         rdev->config.evergreen.tile_config |= 1 << 4;
3378                         break;
3379                 case 2: /* sixteen banks */
3380                 default:
3381                         rdev->config.evergreen.tile_config |= 2 << 4;
3382                         break;
3383                 }
3384         }
3385         rdev->config.evergreen.tile_config |= 0 << 8;
3386         rdev->config.evergreen.tile_config |=
3387                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3388
3389         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3390                 u32 efuse_straps_4;
3391                 u32 efuse_straps_3;
3392
3393                 efuse_straps_4 = RREG32_RCU(0x204);
3394                 efuse_straps_3 = RREG32_RCU(0x203);
3395                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3396                       ((efuse_straps_3 & 0xf0000000) >> 28));
3397         } else {
3398                 tmp = 0;
3399                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3400                         u32 rb_disable_bitmap;
3401
3402                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3403                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3404                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3405                         tmp <<= 4;
3406                         tmp |= rb_disable_bitmap;
3407                 }
3408         }
3409         /* enabled rb are just the one not disabled :) */
3410         disabled_rb_mask = tmp;
3411         tmp = 0;
3412         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3413                 tmp |= (1 << i);
3414         /* if all the backends are disabled, fix it up here */
3415         if ((disabled_rb_mask & tmp) == tmp) {
3416                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3417                         disabled_rb_mask &= ~(1 << i);
3418         }
3419
3420         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3421                 u32 simd_disable_bitmap;
3422
3423                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3424                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3425                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3426                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3427                 tmp <<= 16;
3428                 tmp |= simd_disable_bitmap;
3429         }
3430         rdev->config.evergreen.active_simds = hweight32(~tmp);
3431
3432         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3433         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3434
3435         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3436         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3437         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3438         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3439         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3440         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3441         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3442
3443         if ((rdev->config.evergreen.max_backends == 1) &&
3444             (rdev->flags & RADEON_IS_IGP)) {
3445                 if ((disabled_rb_mask & 3) == 1) {
3446                         /* RB0 disabled, RB1 enabled */
3447                         tmp = 0x11111111;
3448                 } else {
3449                         /* RB1 disabled, RB0 enabled */
3450                         tmp = 0x00000000;
3451                 }
3452         } else {
3453                 tmp = gb_addr_config & NUM_PIPES_MASK;
3454                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3455                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3456         }
3457         WREG32(GB_BACKEND_MAP, tmp);
3458
3459         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3460         WREG32(CGTS_TCC_DISABLE, 0);
3461         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3462         WREG32(CGTS_USER_TCC_DISABLE, 0);
3463
3464         /* set HW defaults for 3D engine */
3465         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3466                                      ROQ_IB2_START(0x2b)));
3467
3468         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3469
3470         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3471                              SYNC_GRADIENT |
3472                              SYNC_WALKER |
3473                              SYNC_ALIGNER));
3474
3475         sx_debug_1 = RREG32(SX_DEBUG_1);
3476         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3477         WREG32(SX_DEBUG_1, sx_debug_1);
3478
3479
3480         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3481         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3482         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3483         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3484
3485         if (rdev->family <= CHIP_SUMO2)
3486                 WREG32(SMX_SAR_CTL0, 0x00010000);
3487
3488         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3489                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3490                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3491
3492         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3493                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3494                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3495
3496         WREG32(VGT_NUM_INSTANCES, 1);
3497         WREG32(SPI_CONFIG_CNTL, 0);
3498         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3499         WREG32(CP_PERFMON_CNTL, 0);
3500
3501         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3502                                   FETCH_FIFO_HIWATER(0x4) |
3503                                   DONE_FIFO_HIWATER(0xe0) |
3504                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3505
3506         sq_config = RREG32(SQ_CONFIG);
3507         sq_config &= ~(PS_PRIO(3) |
3508                        VS_PRIO(3) |
3509                        GS_PRIO(3) |
3510                        ES_PRIO(3));
3511         sq_config |= (VC_ENABLE |
3512                       EXPORT_SRC_C |
3513                       PS_PRIO(0) |
3514                       VS_PRIO(1) |
3515                       GS_PRIO(2) |
3516                       ES_PRIO(3));
3517
3518         switch (rdev->family) {
3519         case CHIP_CEDAR:
3520         case CHIP_PALM:
3521         case CHIP_SUMO:
3522         case CHIP_SUMO2:
3523         case CHIP_CAICOS:
3524                 /* no vertex cache */
3525                 sq_config &= ~VC_ENABLE;
3526                 break;
3527         default:
3528                 break;
3529         }
3530
3531         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3532
3533         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3534         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3535         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3536         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3537         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3538         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3539         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3540
3541         switch (rdev->family) {
3542         case CHIP_CEDAR:
3543         case CHIP_PALM:
3544         case CHIP_SUMO:
3545         case CHIP_SUMO2:
3546                 ps_thread_count = 96;
3547                 break;
3548         default:
3549                 ps_thread_count = 128;
3550                 break;
3551         }
3552
3553         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3554         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3555         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3556         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3557         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3558         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3559
3560         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3561         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3562         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3563         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3564         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3565         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3566
3567         WREG32(SQ_CONFIG, sq_config);
3568         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3569         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3570         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3571         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3572         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3573         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3574         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3575         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3576         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3577         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3578
3579         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3580                                           FORCE_EOV_MAX_REZ_CNT(255)));
3581
3582         switch (rdev->family) {
3583         case CHIP_CEDAR:
3584         case CHIP_PALM:
3585         case CHIP_SUMO:
3586         case CHIP_SUMO2:
3587         case CHIP_CAICOS:
3588                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3589                 break;
3590         default:
3591                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3592                 break;
3593         }
3594         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3595         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3596
3597         WREG32(VGT_GS_VERTEX_REUSE, 16);
3598         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3599         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3600
3601         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3602         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3603
3604         WREG32(CB_PERF_CTR0_SEL_0, 0);
3605         WREG32(CB_PERF_CTR0_SEL_1, 0);
3606         WREG32(CB_PERF_CTR1_SEL_0, 0);
3607         WREG32(CB_PERF_CTR1_SEL_1, 0);
3608         WREG32(CB_PERF_CTR2_SEL_0, 0);
3609         WREG32(CB_PERF_CTR2_SEL_1, 0);
3610         WREG32(CB_PERF_CTR3_SEL_0, 0);
3611         WREG32(CB_PERF_CTR3_SEL_1, 0);
3612
3613         /* clear render buffer base addresses */
3614         WREG32(CB_COLOR0_BASE, 0);
3615         WREG32(CB_COLOR1_BASE, 0);
3616         WREG32(CB_COLOR2_BASE, 0);
3617         WREG32(CB_COLOR3_BASE, 0);
3618         WREG32(CB_COLOR4_BASE, 0);
3619         WREG32(CB_COLOR5_BASE, 0);
3620         WREG32(CB_COLOR6_BASE, 0);
3621         WREG32(CB_COLOR7_BASE, 0);
3622         WREG32(CB_COLOR8_BASE, 0);
3623         WREG32(CB_COLOR9_BASE, 0);
3624         WREG32(CB_COLOR10_BASE, 0);
3625         WREG32(CB_COLOR11_BASE, 0);
3626
3627         /* set the shader const cache sizes to 0 */
3628         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3629                 WREG32(i, 0);
3630         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3631                 WREG32(i, 0);
3632
3633         tmp = RREG32(HDP_MISC_CNTL);
3634         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3635         WREG32(HDP_MISC_CNTL, tmp);
3636
3637         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3638         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3639
3640         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3641
3642         udelay(50);
3643
3644 }
3645
3646 int evergreen_mc_init(struct radeon_device *rdev)
3647 {
3648         u32 tmp;
3649         int chansize, numchan;
3650
3651         /* Get VRAM informations */
3652         rdev->mc.vram_is_ddr = true;
3653         if ((rdev->family == CHIP_PALM) ||
3654             (rdev->family == CHIP_SUMO) ||
3655             (rdev->family == CHIP_SUMO2))
3656                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3657         else
3658                 tmp = RREG32(MC_ARB_RAMCFG);
3659         if (tmp & CHANSIZE_OVERRIDE) {
3660                 chansize = 16;
3661         } else if (tmp & CHANSIZE_MASK) {
3662                 chansize = 64;
3663         } else {
3664                 chansize = 32;
3665         }
3666         tmp = RREG32(MC_SHARED_CHMAP);
3667         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3668         case 0:
3669         default:
3670                 numchan = 1;
3671                 break;
3672         case 1:
3673                 numchan = 2;
3674                 break;
3675         case 2:
3676                 numchan = 4;
3677                 break;
3678         case 3:
3679                 numchan = 8;
3680                 break;
3681         }
3682         rdev->mc.vram_width = numchan * chansize;
3683         /* Could aper size report 0 ? */
3684         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3685         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3686         /* Setup GPU memory space */
3687         if ((rdev->family == CHIP_PALM) ||
3688             (rdev->family == CHIP_SUMO) ||
3689             (rdev->family == CHIP_SUMO2)) {
3690                 /* size in bytes on fusion */
3691                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3692                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3693         } else {
3694                 /* size in MB on evergreen/cayman/tn */
3695                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3696                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3697         }
3698         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3699         r700_vram_gtt_location(rdev, &rdev->mc);
3700         radeon_update_bandwidth_info(rdev);
3701
3702         return 0;
3703 }
3704
3705 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3706 {
3707         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3708                 RREG32(GRBM_STATUS));
3709         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3710                 RREG32(GRBM_STATUS_SE0));
3711         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3712                 RREG32(GRBM_STATUS_SE1));
3713         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3714                 RREG32(SRBM_STATUS));
3715         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3716                 RREG32(SRBM_STATUS2));
3717         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3718                 RREG32(CP_STALLED_STAT1));
3719         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3720                 RREG32(CP_STALLED_STAT2));
3721         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3722                 RREG32(CP_BUSY_STAT));
3723         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3724                 RREG32(CP_STAT));
3725         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3726                 RREG32(DMA_STATUS_REG));
3727         if (rdev->family >= CHIP_CAYMAN) {
3728                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3729                          RREG32(DMA_STATUS_REG + 0x800));
3730         }
3731 }
3732
3733 bool evergreen_is_display_hung(struct radeon_device *rdev)
3734 {
3735         u32 crtc_hung = 0;
3736         u32 crtc_status[6];
3737         u32 i, j, tmp;
3738
3739         for (i = 0; i < rdev->num_crtc; i++) {
3740                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3741                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3742                         crtc_hung |= (1 << i);
3743                 }
3744         }
3745
3746         for (j = 0; j < 10; j++) {
3747                 for (i = 0; i < rdev->num_crtc; i++) {
3748                         if (crtc_hung & (1 << i)) {
3749                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3750                                 if (tmp != crtc_status[i])
3751                                         crtc_hung &= ~(1 << i);
3752                         }
3753                 }
3754                 if (crtc_hung == 0)
3755                         return false;
3756                 udelay(100);
3757         }
3758
3759         return true;
3760 }
3761
3762 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3763 {
3764         u32 reset_mask = 0;
3765         u32 tmp;
3766
3767         /* GRBM_STATUS */
3768         tmp = RREG32(GRBM_STATUS);
3769         if (tmp & (PA_BUSY | SC_BUSY |
3770                    SH_BUSY | SX_BUSY |
3771                    TA_BUSY | VGT_BUSY |
3772                    DB_BUSY | CB_BUSY |
3773                    SPI_BUSY | VGT_BUSY_NO_DMA))
3774                 reset_mask |= RADEON_RESET_GFX;
3775
3776         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3777                    CP_BUSY | CP_COHERENCY_BUSY))
3778                 reset_mask |= RADEON_RESET_CP;
3779
3780         if (tmp & GRBM_EE_BUSY)
3781                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3782
3783         /* DMA_STATUS_REG */
3784         tmp = RREG32(DMA_STATUS_REG);
3785         if (!(tmp & DMA_IDLE))
3786                 reset_mask |= RADEON_RESET_DMA;
3787
3788         /* SRBM_STATUS2 */
3789         tmp = RREG32(SRBM_STATUS2);
3790         if (tmp & DMA_BUSY)
3791                 reset_mask |= RADEON_RESET_DMA;
3792
3793         /* SRBM_STATUS */
3794         tmp = RREG32(SRBM_STATUS);
3795         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3796                 reset_mask |= RADEON_RESET_RLC;
3797
3798         if (tmp & IH_BUSY)
3799                 reset_mask |= RADEON_RESET_IH;
3800
3801         if (tmp & SEM_BUSY)
3802                 reset_mask |= RADEON_RESET_SEM;
3803
3804         if (tmp & GRBM_RQ_PENDING)
3805                 reset_mask |= RADEON_RESET_GRBM;
3806
3807         if (tmp & VMC_BUSY)
3808                 reset_mask |= RADEON_RESET_VMC;
3809
3810         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3811                    MCC_BUSY | MCD_BUSY))
3812                 reset_mask |= RADEON_RESET_MC;
3813
3814         if (evergreen_is_display_hung(rdev))
3815                 reset_mask |= RADEON_RESET_DISPLAY;
3816
3817         /* VM_L2_STATUS */
3818         tmp = RREG32(VM_L2_STATUS);
3819         if (tmp & L2_BUSY)
3820                 reset_mask |= RADEON_RESET_VMC;
3821
3822         /* Skip MC reset as it's mostly likely not hung, just busy */
3823         if (reset_mask & RADEON_RESET_MC) {
3824                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3825                 reset_mask &= ~RADEON_RESET_MC;
3826         }
3827
3828         return reset_mask;
3829 }
3830
3831 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3832 {
3833         struct evergreen_mc_save save;
3834         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3835         u32 tmp;
3836
3837         if (reset_mask == 0)
3838                 return;
3839
3840         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3841
3842         evergreen_print_gpu_status_regs(rdev);
3843
3844         /* Disable CP parsing/prefetching */
3845         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3846
3847         if (reset_mask & RADEON_RESET_DMA) {
3848                 /* Disable DMA */
3849                 tmp = RREG32(DMA_RB_CNTL);
3850                 tmp &= ~DMA_RB_ENABLE;
3851                 WREG32(DMA_RB_CNTL, tmp);
3852         }
3853
3854         udelay(50);
3855
3856         evergreen_mc_stop(rdev, &save);
3857         if (evergreen_mc_wait_for_idle(rdev)) {
3858                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3859         }
3860
3861         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3862                 grbm_soft_reset |= SOFT_RESET_DB |
3863                         SOFT_RESET_CB |
3864                         SOFT_RESET_PA |
3865                         SOFT_RESET_SC |
3866                         SOFT_RESET_SPI |
3867                         SOFT_RESET_SX |
3868                         SOFT_RESET_SH |
3869                         SOFT_RESET_TC |
3870                         SOFT_RESET_TA |
3871                         SOFT_RESET_VC |
3872                         SOFT_RESET_VGT;
3873         }
3874
3875         if (reset_mask & RADEON_RESET_CP) {
3876                 grbm_soft_reset |= SOFT_RESET_CP |
3877                         SOFT_RESET_VGT;
3878
3879                 srbm_soft_reset |= SOFT_RESET_GRBM;
3880         }
3881
3882         if (reset_mask & RADEON_RESET_DMA)
3883                 srbm_soft_reset |= SOFT_RESET_DMA;
3884
3885         if (reset_mask & RADEON_RESET_DISPLAY)
3886                 srbm_soft_reset |= SOFT_RESET_DC;
3887
3888         if (reset_mask & RADEON_RESET_RLC)
3889                 srbm_soft_reset |= SOFT_RESET_RLC;
3890
3891         if (reset_mask & RADEON_RESET_SEM)
3892                 srbm_soft_reset |= SOFT_RESET_SEM;
3893
3894         if (reset_mask & RADEON_RESET_IH)
3895                 srbm_soft_reset |= SOFT_RESET_IH;
3896
3897         if (reset_mask & RADEON_RESET_GRBM)
3898                 srbm_soft_reset |= SOFT_RESET_GRBM;
3899
3900         if (reset_mask & RADEON_RESET_VMC)
3901                 srbm_soft_reset |= SOFT_RESET_VMC;
3902
3903         if (!(rdev->flags & RADEON_IS_IGP)) {
3904                 if (reset_mask & RADEON_RESET_MC)
3905                         srbm_soft_reset |= SOFT_RESET_MC;
3906         }
3907
3908         if (grbm_soft_reset) {
3909                 tmp = RREG32(GRBM_SOFT_RESET);
3910                 tmp |= grbm_soft_reset;
3911                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3912                 WREG32(GRBM_SOFT_RESET, tmp);
3913                 tmp = RREG32(GRBM_SOFT_RESET);
3914
3915                 udelay(50);
3916
3917                 tmp &= ~grbm_soft_reset;
3918                 WREG32(GRBM_SOFT_RESET, tmp);
3919                 tmp = RREG32(GRBM_SOFT_RESET);
3920         }
3921
3922         if (srbm_soft_reset) {
3923                 tmp = RREG32(SRBM_SOFT_RESET);
3924                 tmp |= srbm_soft_reset;
3925                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3926                 WREG32(SRBM_SOFT_RESET, tmp);
3927                 tmp = RREG32(SRBM_SOFT_RESET);
3928
3929                 udelay(50);
3930
3931                 tmp &= ~srbm_soft_reset;
3932                 WREG32(SRBM_SOFT_RESET, tmp);
3933                 tmp = RREG32(SRBM_SOFT_RESET);
3934         }
3935
3936         /* Wait a little for things to settle down */
3937         udelay(50);
3938
3939         evergreen_mc_resume(rdev, &save);
3940         udelay(50);
3941
3942         evergreen_print_gpu_status_regs(rdev);
3943 }
3944
3945 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3946 {
3947         struct evergreen_mc_save save;
3948         u32 tmp, i;
3949
3950         dev_info(rdev->dev, "GPU pci config reset\n");
3951
3952         /* disable dpm? */
3953
3954         /* Disable CP parsing/prefetching */
3955         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3956         udelay(50);
3957         /* Disable DMA */
3958         tmp = RREG32(DMA_RB_CNTL);
3959         tmp &= ~DMA_RB_ENABLE;
3960         WREG32(DMA_RB_CNTL, tmp);
3961         /* XXX other engines? */
3962
3963         /* halt the rlc */
3964         r600_rlc_stop(rdev);
3965
3966         udelay(50);
3967
3968         /* set mclk/sclk to bypass */
3969         rv770_set_clk_bypass_mode(rdev);
3970         /* disable BM */
3971         pci_clear_master(rdev->pdev);
3972         /* disable mem access */
3973         evergreen_mc_stop(rdev, &save);
3974         if (evergreen_mc_wait_for_idle(rdev)) {
3975                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3976         }
3977         /* reset */
3978         radeon_pci_config_reset(rdev);
3979         /* wait for asic to come out of reset */
3980         for (i = 0; i < rdev->usec_timeout; i++) {
3981                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3982                         break;
3983                 udelay(1);
3984         }
3985 }
3986
3987 int evergreen_asic_reset(struct radeon_device *rdev)
3988 {
3989         u32 reset_mask;
3990
3991         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3992
3993         if (reset_mask)
3994                 r600_set_bios_scratch_engine_hung(rdev, true);
3995
3996         /* try soft reset */
3997         evergreen_gpu_soft_reset(rdev, reset_mask);
3998
3999         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4000
4001         /* try pci config reset */
4002         if (reset_mask && radeon_hard_reset)
4003                 evergreen_gpu_pci_config_reset(rdev);
4004
4005         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4006
4007         if (!reset_mask)
4008                 r600_set_bios_scratch_engine_hung(rdev, false);
4009
4010         return 0;
4011 }
4012
4013 /**
4014  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4015  *
4016  * @rdev: radeon_device pointer
4017  * @ring: radeon_ring structure holding ring information
4018  *
4019  * Check if the GFX engine is locked up.
4020  * Returns true if the engine appears to be locked up, false if not.
4021  */
4022 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4023 {
4024         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4025
4026         if (!(reset_mask & (RADEON_RESET_GFX |
4027                             RADEON_RESET_COMPUTE |
4028                             RADEON_RESET_CP))) {
4029                 radeon_ring_lockup_update(rdev, ring);
4030                 return false;
4031         }
4032         return radeon_ring_test_lockup(rdev, ring);
4033 }
4034
4035 /*
4036  * RLC
4037  */
4038 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4039 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4040
4041 void sumo_rlc_fini(struct radeon_device *rdev)
4042 {
4043         int r;
4044
4045         /* save restore block */
4046         if (rdev->rlc.save_restore_obj) {
4047                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4048                 if (unlikely(r != 0))
4049                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4050                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4051                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4052
4053                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4054                 rdev->rlc.save_restore_obj = NULL;
4055         }
4056
4057         /* clear state block */
4058         if (rdev->rlc.clear_state_obj) {
4059                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4060                 if (unlikely(r != 0))
4061                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4062                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4063                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4064
4065                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4066                 rdev->rlc.clear_state_obj = NULL;
4067         }
4068
4069         /* clear state block */
4070         if (rdev->rlc.cp_table_obj) {
4071                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4072                 if (unlikely(r != 0))
4073                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4074                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4075                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4076
4077                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4078                 rdev->rlc.cp_table_obj = NULL;
4079         }
4080 }
4081
4082 #define CP_ME_TABLE_SIZE    96
4083
4084 int sumo_rlc_init(struct radeon_device *rdev)
4085 {
4086         const u32 *src_ptr;
4087         volatile u32 *dst_ptr;
4088         u32 dws, data, i, j, k, reg_num;
4089         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4090         u64 reg_list_mc_addr;
4091         const struct cs_section_def *cs_data;
4092         int r;
4093
4094         src_ptr = rdev->rlc.reg_list;
4095         dws = rdev->rlc.reg_list_size;
4096         if (rdev->family >= CHIP_BONAIRE) {
4097                 dws += (5 * 16) + 48 + 48 + 64;
4098         }
4099         cs_data = rdev->rlc.cs_data;
4100
4101         if (src_ptr) {
4102                 /* save restore block */
4103                 if (rdev->rlc.save_restore_obj == NULL) {
4104                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4105                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4106                                              NULL, &rdev->rlc.save_restore_obj);
4107                         if (r) {
4108                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4109                                 return r;
4110                         }
4111                 }
4112
4113                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4114                 if (unlikely(r != 0)) {
4115                         sumo_rlc_fini(rdev);
4116                         return r;
4117                 }
4118                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4119                                   &rdev->rlc.save_restore_gpu_addr);
4120                 if (r) {
4121                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4122                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4123                         sumo_rlc_fini(rdev);
4124                         return r;
4125                 }
4126
4127                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4128                 if (r) {
4129                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4130                         sumo_rlc_fini(rdev);
4131                         return r;
4132                 }
4133                 /* write the sr buffer */
4134                 dst_ptr = rdev->rlc.sr_ptr;
4135                 if (rdev->family >= CHIP_TAHITI) {
4136                         /* SI */
4137                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4138                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4139                 } else {
4140                         /* ON/LN/TN */
4141                         /* format:
4142                          * dw0: (reg2 << 16) | reg1
4143                          * dw1: reg1 save space
4144                          * dw2: reg2 save space
4145                          */
4146                         for (i = 0; i < dws; i++) {
4147                                 data = src_ptr[i] >> 2;
4148                                 i++;
4149                                 if (i < dws)
4150                                         data |= (src_ptr[i] >> 2) << 16;
4151                                 j = (((i - 1) * 3) / 2);
4152                                 dst_ptr[j] = cpu_to_le32(data);
4153                         }
4154                         j = ((i * 3) / 2);
4155                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4156                 }
4157                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4158                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4159         }
4160
4161         if (cs_data) {
4162                 /* clear state block */
4163                 if (rdev->family >= CHIP_BONAIRE) {
4164                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4165                 } else if (rdev->family >= CHIP_TAHITI) {
4166                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4167                         dws = rdev->rlc.clear_state_size + (256 / 4);
4168                 } else {
4169                         reg_list_num = 0;
4170                         dws = 0;
4171                         for (i = 0; cs_data[i].section != NULL; i++) {
4172                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4173                                         reg_list_num++;
4174                                         dws += cs_data[i].section[j].reg_count;
4175                                 }
4176                         }
4177                         reg_list_blk_index = (3 * reg_list_num + 2);
4178                         dws += reg_list_blk_index;
4179                         rdev->rlc.clear_state_size = dws;
4180                 }
4181
4182                 if (rdev->rlc.clear_state_obj == NULL) {
4183                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4184                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4185                                              NULL, &rdev->rlc.clear_state_obj);
4186                         if (r) {
4187                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4188                                 sumo_rlc_fini(rdev);
4189                                 return r;
4190                         }
4191                 }
4192                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4193                 if (unlikely(r != 0)) {
4194                         sumo_rlc_fini(rdev);
4195                         return r;
4196                 }
4197                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4198                                   &rdev->rlc.clear_state_gpu_addr);
4199                 if (r) {
4200                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4201                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4202                         sumo_rlc_fini(rdev);
4203                         return r;
4204                 }
4205
4206                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4207                 if (r) {
4208                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4209                         sumo_rlc_fini(rdev);
4210                         return r;
4211                 }
4212                 /* set up the cs buffer */
4213                 dst_ptr = rdev->rlc.cs_ptr;
4214                 if (rdev->family >= CHIP_BONAIRE) {
4215                         cik_get_csb_buffer(rdev, dst_ptr);
4216                 } else if (rdev->family >= CHIP_TAHITI) {
4217                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4218                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4219                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4220                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4221                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4222                 } else {
4223                         reg_list_hdr_blk_index = 0;
4224                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4225                         data = upper_32_bits(reg_list_mc_addr);
4226                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4227                         reg_list_hdr_blk_index++;
4228                         for (i = 0; cs_data[i].section != NULL; i++) {
4229                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4230                                         reg_num = cs_data[i].section[j].reg_count;
4231                                         data = reg_list_mc_addr & 0xffffffff;
4232                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4233                                         reg_list_hdr_blk_index++;
4234
4235                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4236                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4237                                         reg_list_hdr_blk_index++;
4238
4239                                         data = 0x08000000 | (reg_num * 4);
4240                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4241                                         reg_list_hdr_blk_index++;
4242
4243                                         for (k = 0; k < reg_num; k++) {
4244                                                 data = cs_data[i].section[j].extent[k];
4245                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4246                                         }
4247                                         reg_list_mc_addr += reg_num * 4;
4248                                         reg_list_blk_index += reg_num;
4249                                 }
4250                         }
4251                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4252                 }
4253                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4254                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4255         }
4256
4257         if (rdev->rlc.cp_table_size) {
4258                 if (rdev->rlc.cp_table_obj == NULL) {
4259                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4260                                              PAGE_SIZE, true,
4261                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4262                                              NULL, &rdev->rlc.cp_table_obj);
4263                         if (r) {
4264                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4265                                 sumo_rlc_fini(rdev);
4266                                 return r;
4267                         }
4268                 }
4269
4270                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4271                 if (unlikely(r != 0)) {
4272                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4273                         sumo_rlc_fini(rdev);
4274                         return r;
4275                 }
4276                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4277                                   &rdev->rlc.cp_table_gpu_addr);
4278                 if (r) {
4279                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4280                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4281                         sumo_rlc_fini(rdev);
4282                         return r;
4283                 }
4284                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4285                 if (r) {
4286                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4287                         sumo_rlc_fini(rdev);
4288                         return r;
4289                 }
4290
4291                 cik_init_cp_pg_table(rdev);
4292
4293                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4294                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4295
4296         }
4297
4298         return 0;
4299 }
4300
4301 static void evergreen_rlc_start(struct radeon_device *rdev)
4302 {
4303         u32 mask = RLC_ENABLE;
4304
4305         if (rdev->flags & RADEON_IS_IGP) {
4306                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4307         }
4308
4309         WREG32(RLC_CNTL, mask);
4310 }
4311
4312 int evergreen_rlc_resume(struct radeon_device *rdev)
4313 {
4314         u32 i;
4315         const __be32 *fw_data;
4316
4317         if (!rdev->rlc_fw)
4318                 return -EINVAL;
4319
4320         r600_rlc_stop(rdev);
4321
4322         WREG32(RLC_HB_CNTL, 0);
4323
4324         if (rdev->flags & RADEON_IS_IGP) {
4325                 if (rdev->family == CHIP_ARUBA) {
4326                         u32 always_on_bitmap =
4327                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4328                         /* find out the number of active simds */
4329                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4330                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4331                         tmp = hweight32(~tmp);
4332                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4333                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4334                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4335                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4336                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4337                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4338                         }
4339                 } else {
4340                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4341                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4342                 }
4343                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4344                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4345         } else {
4346                 WREG32(RLC_HB_BASE, 0);
4347                 WREG32(RLC_HB_RPTR, 0);
4348                 WREG32(RLC_HB_WPTR, 0);
4349                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4350                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4351         }
4352         WREG32(RLC_MC_CNTL, 0);
4353         WREG32(RLC_UCODE_CNTL, 0);
4354
4355         fw_data = (const __be32 *)rdev->rlc_fw->data;
4356         if (rdev->family >= CHIP_ARUBA) {
4357                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4358                         WREG32(RLC_UCODE_ADDR, i);
4359                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4360                 }
4361         } else if (rdev->family >= CHIP_CAYMAN) {
4362                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4363                         WREG32(RLC_UCODE_ADDR, i);
4364                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4365                 }
4366         } else {
4367                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4368                         WREG32(RLC_UCODE_ADDR, i);
4369                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4370                 }
4371         }
4372         WREG32(RLC_UCODE_ADDR, 0);
4373
4374         evergreen_rlc_start(rdev);
4375
4376         return 0;
4377 }
4378
4379 /* Interrupts */
4380
4381 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4382 {
4383         if (crtc >= rdev->num_crtc)
4384                 return 0;
4385         else
4386                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4387 }
4388
4389 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4390 {
4391         u32 tmp;
4392
4393         if (rdev->family >= CHIP_CAYMAN) {
4394                 cayman_cp_int_cntl_setup(rdev, 0,
4395                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4396                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4397                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4398                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4399                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4400         } else
4401                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4402         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4403         WREG32(DMA_CNTL, tmp);
4404         WREG32(GRBM_INT_CNTL, 0);
4405         WREG32(SRBM_INT_CNTL, 0);
4406         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4407         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4408         if (rdev->num_crtc >= 4) {
4409                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4410                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4411         }
4412         if (rdev->num_crtc >= 6) {
4413                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4414                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4415         }
4416
4417         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4418         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4419         if (rdev->num_crtc >= 4) {
4420                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4421                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4422         }
4423         if (rdev->num_crtc >= 6) {
4424                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4425                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4426         }
4427
4428         /* only one DAC on DCE5 */
4429         if (!ASIC_IS_DCE5(rdev))
4430                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4431         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4432
4433         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4434         WREG32(DC_HPD1_INT_CONTROL, tmp);
4435         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4436         WREG32(DC_HPD2_INT_CONTROL, tmp);
4437         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4438         WREG32(DC_HPD3_INT_CONTROL, tmp);
4439         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4440         WREG32(DC_HPD4_INT_CONTROL, tmp);
4441         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4442         WREG32(DC_HPD5_INT_CONTROL, tmp);
4443         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4444         WREG32(DC_HPD6_INT_CONTROL, tmp);
4445
4446 }
4447
4448 int evergreen_irq_set(struct radeon_device *rdev)
4449 {
4450         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4451         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4452         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4453         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4454         u32 grbm_int_cntl = 0;
4455         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4456         u32 dma_cntl, dma_cntl1 = 0;
4457         u32 thermal_int = 0;
4458
4459         if (!rdev->irq.installed) {
4460                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4461                 return -EINVAL;
4462         }
4463         /* don't enable anything if the ih is disabled */
4464         if (!rdev->ih.enabled) {
4465                 r600_disable_interrupts(rdev);
4466                 /* force the active interrupt state to all disabled */
4467                 evergreen_disable_interrupt_state(rdev);
4468                 return 0;
4469         }
4470
4471         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4472         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4473         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4474         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4475         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4476         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4477         if (rdev->family == CHIP_ARUBA)
4478                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4479                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4480         else
4481                 thermal_int = RREG32(CG_THERMAL_INT) &
4482                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4483
4484         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4485         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4486         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4487         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4488         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4489         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4490
4491         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4492
4493         if (rdev->family >= CHIP_CAYMAN) {
4494                 /* enable CP interrupts on all rings */
4495                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4496                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4497                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4498                 }
4499                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4500                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4501                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4502                 }
4503                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4504                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4505                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4506                 }
4507         } else {
4508                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4509                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4510                         cp_int_cntl |= RB_INT_ENABLE;
4511                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4512                 }
4513         }
4514
4515         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4516                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4517                 dma_cntl |= TRAP_ENABLE;
4518         }
4519
4520         if (rdev->family >= CHIP_CAYMAN) {
4521                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4522                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4523                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4524                         dma_cntl1 |= TRAP_ENABLE;
4525                 }
4526         }
4527
4528         if (rdev->irq.dpm_thermal) {
4529                 DRM_DEBUG("dpm thermal\n");
4530                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4531         }
4532
4533         if (rdev->irq.crtc_vblank_int[0] ||
4534             atomic_read(&rdev->irq.pflip[0])) {
4535                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4536                 crtc1 |= VBLANK_INT_MASK;
4537         }
4538         if (rdev->irq.crtc_vblank_int[1] ||
4539             atomic_read(&rdev->irq.pflip[1])) {
4540                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4541                 crtc2 |= VBLANK_INT_MASK;
4542         }
4543         if (rdev->irq.crtc_vblank_int[2] ||
4544             atomic_read(&rdev->irq.pflip[2])) {
4545                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4546                 crtc3 |= VBLANK_INT_MASK;
4547         }
4548         if (rdev->irq.crtc_vblank_int[3] ||
4549             atomic_read(&rdev->irq.pflip[3])) {
4550                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4551                 crtc4 |= VBLANK_INT_MASK;
4552         }
4553         if (rdev->irq.crtc_vblank_int[4] ||
4554             atomic_read(&rdev->irq.pflip[4])) {
4555                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4556                 crtc5 |= VBLANK_INT_MASK;
4557         }
4558         if (rdev->irq.crtc_vblank_int[5] ||
4559             atomic_read(&rdev->irq.pflip[5])) {
4560                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4561                 crtc6 |= VBLANK_INT_MASK;
4562         }
4563         if (rdev->irq.hpd[0]) {
4564                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4565                 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4566         }
4567         if (rdev->irq.hpd[1]) {
4568                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4569                 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4570         }
4571         if (rdev->irq.hpd[2]) {
4572                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4573                 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4574         }
4575         if (rdev->irq.hpd[3]) {
4576                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4577                 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4578         }
4579         if (rdev->irq.hpd[4]) {
4580                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4581                 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4582         }
4583         if (rdev->irq.hpd[5]) {
4584                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4585                 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4586         }
4587         if (rdev->irq.afmt[0]) {
4588                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4589                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4590         }
4591         if (rdev->irq.afmt[1]) {
4592                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4593                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4594         }
4595         if (rdev->irq.afmt[2]) {
4596                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4597                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4598         }
4599         if (rdev->irq.afmt[3]) {
4600                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4601                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4602         }
4603         if (rdev->irq.afmt[4]) {
4604                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4605                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4606         }
4607         if (rdev->irq.afmt[5]) {
4608                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4609                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4610         }
4611
4612         if (rdev->family >= CHIP_CAYMAN) {
4613                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4614                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4615                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4616         } else
4617                 WREG32(CP_INT_CNTL, cp_int_cntl);
4618
4619         WREG32(DMA_CNTL, dma_cntl);
4620
4621         if (rdev->family >= CHIP_CAYMAN)
4622                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4623
4624         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4625
4626         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4627         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4628         if (rdev->num_crtc >= 4) {
4629                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4630                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4631         }
4632         if (rdev->num_crtc >= 6) {
4633                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4634                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4635         }
4636
4637         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4638                GRPH_PFLIP_INT_MASK);
4639         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4640                GRPH_PFLIP_INT_MASK);
4641         if (rdev->num_crtc >= 4) {
4642                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4643                        GRPH_PFLIP_INT_MASK);
4644                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4645                        GRPH_PFLIP_INT_MASK);
4646         }
4647         if (rdev->num_crtc >= 6) {
4648                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4649                        GRPH_PFLIP_INT_MASK);
4650                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4651                        GRPH_PFLIP_INT_MASK);
4652         }
4653
4654         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4655         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4656         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4657         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4658         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4659         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4660         if (rdev->family == CHIP_ARUBA)
4661                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4662         else
4663                 WREG32(CG_THERMAL_INT, thermal_int);
4664
4665         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4666         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4667         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4668         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4669         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4670         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4671
4672         /* posting read */
4673         RREG32(SRBM_STATUS);
4674
4675         return 0;
4676 }
4677
4678 static void evergreen_irq_ack(struct radeon_device *rdev)
4679 {
4680         u32 tmp;
4681
4682         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4683         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4684         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4685         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4686         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4687         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4688         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4689         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4690         if (rdev->num_crtc >= 4) {
4691                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4692                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4693         }
4694         if (rdev->num_crtc >= 6) {
4695                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4696                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4697         }
4698
4699         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4700         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4701         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4702         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4703         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4704         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4705
4706         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4707                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4708         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4709                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4710         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4711                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4712         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4713                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4714         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4715                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4716         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4717                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4718
4719         if (rdev->num_crtc >= 4) {
4720                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4721                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4722                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4723                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4724                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4725                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4726                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4727                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4728                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4729                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4730                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4731                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4732         }
4733
4734         if (rdev->num_crtc >= 6) {
4735                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4736                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4737                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4738                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4739                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4740                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4741                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4742                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4743                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4744                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4745                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4746                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4747         }
4748
4749         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4750                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4751                 tmp |= DC_HPDx_INT_ACK;
4752                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4753         }
4754         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4755                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4756                 tmp |= DC_HPDx_INT_ACK;
4757                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4758         }
4759         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4760                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4761                 tmp |= DC_HPDx_INT_ACK;
4762                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4763         }
4764         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4765                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4766                 tmp |= DC_HPDx_INT_ACK;
4767                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4768         }
4769         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4770                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4771                 tmp |= DC_HPDx_INT_ACK;
4772                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4773         }
4774         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4775                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4776                 tmp |= DC_HPDx_INT_ACK;
4777                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4778         }
4779
4780         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4781                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4782                 tmp |= DC_HPDx_RX_INT_ACK;
4783                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4784         }
4785         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4786                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4787                 tmp |= DC_HPDx_RX_INT_ACK;
4788                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4789         }
4790         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4791                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4792                 tmp |= DC_HPDx_RX_INT_ACK;
4793                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4794         }
4795         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4796                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4797                 tmp |= DC_HPDx_RX_INT_ACK;
4798                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4799         }
4800         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4801                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4802                 tmp |= DC_HPDx_RX_INT_ACK;
4803                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4804         }
4805         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4806                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4807                 tmp |= DC_HPDx_RX_INT_ACK;
4808                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4809         }
4810
4811         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4812                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4813                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4814                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4815         }
4816         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4817                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4818                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4819                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4820         }
4821         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4822                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4823                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4824                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4825         }
4826         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4827                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4828                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4829                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4830         }
4831         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4832                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4833                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4834                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4835         }
4836         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4837                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4838                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4839                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4840         }
4841 }
4842
4843 static void evergreen_irq_disable(struct radeon_device *rdev)
4844 {
4845         r600_disable_interrupts(rdev);
4846         /* Wait and acknowledge irq */
4847         mdelay(1);
4848         evergreen_irq_ack(rdev);
4849         evergreen_disable_interrupt_state(rdev);
4850 }
4851
4852 void evergreen_irq_suspend(struct radeon_device *rdev)
4853 {
4854         evergreen_irq_disable(rdev);
4855         r600_rlc_stop(rdev);
4856 }
4857
4858 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4859 {
4860         u32 wptr, tmp;
4861
4862         if (rdev->wb.enabled)
4863                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4864         else
4865                 wptr = RREG32(IH_RB_WPTR);
4866
4867         if (wptr & RB_OVERFLOW) {
4868                 wptr &= ~RB_OVERFLOW;
4869                 /* When a ring buffer overflow happen start parsing interrupt
4870                  * from the last not overwritten vector (wptr + 16). Hopefully
4871                  * this should allow us to catchup.
4872                  */
4873                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4874                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4875                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4876                 tmp = RREG32(IH_RB_CNTL);
4877                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4878                 WREG32(IH_RB_CNTL, tmp);
4879         }
4880         return (wptr & rdev->ih.ptr_mask);
4881 }
4882
4883 int evergreen_irq_process(struct radeon_device *rdev)
4884 {
4885         u32 wptr;
4886         u32 rptr;
4887         u32 src_id, src_data;
4888         u32 ring_index;
4889         bool queue_hotplug = false;
4890         bool queue_hdmi = false;
4891         bool queue_dp = false;
4892         bool queue_thermal = false;
4893         u32 status, addr;
4894
4895         if (!rdev->ih.enabled || rdev->shutdown)
4896                 return IRQ_NONE;
4897
4898         wptr = evergreen_get_ih_wptr(rdev);
4899
4900 restart_ih:
4901         /* is somebody else already processing irqs? */
4902         if (atomic_xchg(&rdev->ih.lock, 1))
4903                 return IRQ_NONE;
4904
4905         rptr = rdev->ih.rptr;
4906         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4907
4908         /* Order reading of wptr vs. reading of IH ring data */
4909         rmb();
4910
4911         /* display interrupts */
4912         evergreen_irq_ack(rdev);
4913
4914         while (rptr != wptr) {
4915                 /* wptr/rptr are in bytes! */
4916                 ring_index = rptr / 4;
4917                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4918                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4919
4920                 switch (src_id) {
4921                 case 1: /* D1 vblank/vline */
4922                         switch (src_data) {
4923                         case 0: /* D1 vblank */
4924                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
4925                                         DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4926
4927                                 if (rdev->irq.crtc_vblank_int[0]) {
4928                                         drm_handle_vblank(rdev->ddev, 0);
4929                                         rdev->pm.vblank_sync = true;
4930                                         wake_up(&rdev->irq.vblank_queue);
4931                                 }
4932                                 if (atomic_read(&rdev->irq.pflip[0]))
4933                                         radeon_crtc_handle_vblank(rdev, 0);
4934                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4935                                 DRM_DEBUG("IH: D1 vblank\n");
4936
4937                                 break;
4938                         case 1: /* D1 vline */
4939                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
4940                                         DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4941
4942                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4943                                 DRM_DEBUG("IH: D1 vline\n");
4944
4945                                 break;
4946                         default:
4947                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4948                                 break;
4949                         }
4950                         break;
4951                 case 2: /* D2 vblank/vline */
4952                         switch (src_data) {
4953                         case 0: /* D2 vblank */
4954                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
4955                                         DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4956
4957                                 if (rdev->irq.crtc_vblank_int[1]) {
4958                                         drm_handle_vblank(rdev->ddev, 1);
4959                                         rdev->pm.vblank_sync = true;
4960                                         wake_up(&rdev->irq.vblank_queue);
4961                                 }
4962                                 if (atomic_read(&rdev->irq.pflip[1]))
4963                                         radeon_crtc_handle_vblank(rdev, 1);
4964                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4965                                 DRM_DEBUG("IH: D2 vblank\n");
4966
4967                                 break;
4968                         case 1: /* D2 vline */
4969                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
4970                                         DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4971
4972                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4973                                 DRM_DEBUG("IH: D2 vline\n");
4974
4975                                 break;
4976                         default:
4977                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4978                                 break;
4979                         }
4980                         break;
4981                 case 3: /* D3 vblank/vline */
4982                         switch (src_data) {
4983                         case 0: /* D3 vblank */
4984                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
4985                                         DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
4986
4987                                 if (rdev->irq.crtc_vblank_int[2]) {
4988                                         drm_handle_vblank(rdev->ddev, 2);
4989                                         rdev->pm.vblank_sync = true;
4990                                         wake_up(&rdev->irq.vblank_queue);
4991                                 }
4992                                 if (atomic_read(&rdev->irq.pflip[2]))
4993                                         radeon_crtc_handle_vblank(rdev, 2);
4994                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4995                                 DRM_DEBUG("IH: D3 vblank\n");
4996
4997                                 break;
4998                         case 1: /* D3 vline */
4999                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5000                                         DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5001
5002                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5003                                 DRM_DEBUG("IH: D3 vline\n");
5004
5005                                 break;
5006                         default:
5007                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5008                                 break;
5009                         }
5010                         break;
5011                 case 4: /* D4 vblank/vline */
5012                         switch (src_data) {
5013                         case 0: /* D4 vblank */
5014                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5015                                         DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5016
5017                                 if (rdev->irq.crtc_vblank_int[3]) {
5018                                         drm_handle_vblank(rdev->ddev, 3);
5019                                         rdev->pm.vblank_sync = true;
5020                                         wake_up(&rdev->irq.vblank_queue);
5021                                 }
5022                                 if (atomic_read(&rdev->irq.pflip[3]))
5023                                         radeon_crtc_handle_vblank(rdev, 3);
5024                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5025                                 DRM_DEBUG("IH: D4 vblank\n");
5026
5027                                 break;
5028                         case 1: /* D4 vline */
5029                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5030                                         DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5031
5032                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5033                                 DRM_DEBUG("IH: D4 vline\n");
5034
5035                                 break;
5036                         default:
5037                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5038                                 break;
5039                         }
5040                         break;
5041                 case 5: /* D5 vblank/vline */
5042                         switch (src_data) {
5043                         case 0: /* D5 vblank */
5044                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5045                                         DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5046
5047                                 if (rdev->irq.crtc_vblank_int[4]) {
5048                                         drm_handle_vblank(rdev->ddev, 4);
5049                                         rdev->pm.vblank_sync = true;
5050                                         wake_up(&rdev->irq.vblank_queue);
5051                                 }
5052                                 if (atomic_read(&rdev->irq.pflip[4]))
5053                                         radeon_crtc_handle_vblank(rdev, 4);
5054                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5055                                 DRM_DEBUG("IH: D5 vblank\n");
5056
5057                                 break;
5058                         case 1: /* D5 vline */
5059                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5060                                         DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5061
5062                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5063                                 DRM_DEBUG("IH: D5 vline\n");
5064
5065                                 break;
5066                         default:
5067                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5068                                 break;
5069                         }
5070                         break;
5071                 case 6: /* D6 vblank/vline */
5072                         switch (src_data) {
5073                         case 0: /* D6 vblank */
5074                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5075                                         DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5076
5077                                 if (rdev->irq.crtc_vblank_int[5]) {
5078                                         drm_handle_vblank(rdev->ddev, 5);
5079                                         rdev->pm.vblank_sync = true;
5080                                         wake_up(&rdev->irq.vblank_queue);
5081                                 }
5082                                 if (atomic_read(&rdev->irq.pflip[5]))
5083                                         radeon_crtc_handle_vblank(rdev, 5);
5084                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5085                                 DRM_DEBUG("IH: D6 vblank\n");
5086
5087                                 break;
5088                         case 1: /* D6 vline */
5089                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5090                                         DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5091
5092                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5093                                 DRM_DEBUG("IH: D6 vline\n");
5094
5095                                 break;
5096                         default:
5097                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5098                                 break;
5099                         }
5100                         break;
5101                 case 8: /* D1 page flip */
5102                 case 10: /* D2 page flip */
5103                 case 12: /* D3 page flip */
5104                 case 14: /* D4 page flip */
5105                 case 16: /* D5 page flip */
5106                 case 18: /* D6 page flip */
5107                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5108                         if (radeon_use_pflipirq > 0)
5109                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5110                         break;
5111                 case 42: /* HPD hotplug */
5112                         switch (src_data) {
5113                         case 0:
5114                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5115                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5116
5117                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5118                                 queue_hotplug = true;
5119                                 DRM_DEBUG("IH: HPD1\n");
5120                                 break;
5121                         case 1:
5122                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5123                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5124
5125                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5126                                 queue_hotplug = true;
5127                                 DRM_DEBUG("IH: HPD2\n");
5128                                 break;
5129                         case 2:
5130                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5131                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5132
5133                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5134                                 queue_hotplug = true;
5135                                 DRM_DEBUG("IH: HPD3\n");
5136                                 break;
5137                         case 3:
5138                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5139                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5140
5141                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5142                                 queue_hotplug = true;
5143                                 DRM_DEBUG("IH: HPD4\n");
5144                                 break;
5145                         case 4:
5146                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5147                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5148
5149                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5150                                 queue_hotplug = true;
5151                                 DRM_DEBUG("IH: HPD5\n");
5152                                 break;
5153                         case 5:
5154                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5155                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5156
5157                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5158                                 queue_hotplug = true;
5159                                 DRM_DEBUG("IH: HPD6\n");
5160                                 break;
5161                         case 6:
5162                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5163                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5164
5165                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5166                                 queue_dp = true;
5167                                 DRM_DEBUG("IH: HPD_RX 1\n");
5168                                 break;
5169                         case 7:
5170                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5171                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5172
5173                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5174                                 queue_dp = true;
5175                                 DRM_DEBUG("IH: HPD_RX 2\n");
5176                                 break;
5177                         case 8:
5178                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5179                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5180
5181                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5182                                 queue_dp = true;
5183                                 DRM_DEBUG("IH: HPD_RX 3\n");
5184                                 break;
5185                         case 9:
5186                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5187                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5188
5189                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5190                                 queue_dp = true;
5191                                 DRM_DEBUG("IH: HPD_RX 4\n");
5192                                 break;
5193                         case 10:
5194                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5195                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5196
5197                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5198                                 queue_dp = true;
5199                                 DRM_DEBUG("IH: HPD_RX 5\n");
5200                                 break;
5201                         case 11:
5202                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5203                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5204
5205                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5206                                 queue_dp = true;
5207                                 DRM_DEBUG("IH: HPD_RX 6\n");
5208                                 break;
5209                         default:
5210                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5211                                 break;
5212                         }
5213                         break;
5214                 case 44: /* hdmi */
5215                         switch (src_data) {
5216                         case 0:
5217                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5218                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5219
5220                                 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5221                                 queue_hdmi = true;
5222                                 DRM_DEBUG("IH: HDMI0\n");
5223                                 break;
5224                         case 1:
5225                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5226                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5227
5228                                 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5229                                 queue_hdmi = true;
5230                                 DRM_DEBUG("IH: HDMI1\n");
5231                                 break;
5232                         case 2:
5233                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5234                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5235
5236                                 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5237                                 queue_hdmi = true;
5238                                 DRM_DEBUG("IH: HDMI2\n");
5239                                 break;
5240                         case 3:
5241                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5242                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5243
5244                                 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5245                                 queue_hdmi = true;
5246                                 DRM_DEBUG("IH: HDMI3\n");
5247                                 break;
5248                         case 4:
5249                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5250                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5251
5252                                 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5253                                 queue_hdmi = true;
5254                                 DRM_DEBUG("IH: HDMI4\n");
5255                                 break;
5256                         case 5:
5257                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5258                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5259
5260                                 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5261                                 queue_hdmi = true;
5262                                 DRM_DEBUG("IH: HDMI5\n");
5263                                 break;
5264                         default:
5265                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5266                                 break;
5267                         }
5268                 case 96:
5269                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5270                         WREG32(SRBM_INT_ACK, 0x1);
5271                         break;
5272                 case 124: /* UVD */
5273                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5274                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5275                         break;
5276                 case 146:
5277                 case 147:
5278                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5279                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5280                         /* reset addr and status */
5281                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5282                         if (addr == 0x0 && status == 0x0)
5283                                 break;
5284                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5285                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5286                                 addr);
5287                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5288                                 status);
5289                         cayman_vm_decode_fault(rdev, status, addr);
5290                         break;
5291                 case 176: /* CP_INT in ring buffer */
5292                 case 177: /* CP_INT in IB1 */
5293                 case 178: /* CP_INT in IB2 */
5294                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5295                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5296                         break;
5297                 case 181: /* CP EOP event */
5298                         DRM_DEBUG("IH: CP EOP\n");
5299                         if (rdev->family >= CHIP_CAYMAN) {
5300                                 switch (src_data) {
5301                                 case 0:
5302                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5303                                         break;
5304                                 case 1:
5305                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5306                                         break;
5307                                 case 2:
5308                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5309                                         break;
5310                                 }
5311                         } else
5312                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5313                         break;
5314                 case 224: /* DMA trap event */
5315                         DRM_DEBUG("IH: DMA trap\n");
5316                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5317                         break;
5318                 case 230: /* thermal low to high */
5319                         DRM_DEBUG("IH: thermal low to high\n");
5320                         rdev->pm.dpm.thermal.high_to_low = false;
5321                         queue_thermal = true;
5322                         break;
5323                 case 231: /* thermal high to low */
5324                         DRM_DEBUG("IH: thermal high to low\n");
5325                         rdev->pm.dpm.thermal.high_to_low = true;
5326                         queue_thermal = true;
5327                         break;
5328                 case 233: /* GUI IDLE */
5329                         DRM_DEBUG("IH: GUI idle\n");
5330                         break;
5331                 case 244: /* DMA trap event */
5332                         if (rdev->family >= CHIP_CAYMAN) {
5333                                 DRM_DEBUG("IH: DMA1 trap\n");
5334                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5335                         }
5336                         break;
5337                 default:
5338                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5339                         break;
5340                 }
5341
5342                 /* wptr/rptr are in bytes! */
5343                 rptr += 16;
5344                 rptr &= rdev->ih.ptr_mask;
5345                 WREG32(IH_RB_RPTR, rptr);
5346         }
5347         if (queue_dp)
5348                 schedule_work(&rdev->dp_work);
5349         if (queue_hotplug)
5350                 schedule_delayed_work(&rdev->hotplug_work, 0);
5351         if (queue_hdmi)
5352                 schedule_work(&rdev->audio_work);
5353         if (queue_thermal && rdev->pm.dpm_enabled)
5354                 schedule_work(&rdev->pm.dpm.thermal.work);
5355         rdev->ih.rptr = rptr;
5356         atomic_set(&rdev->ih.lock, 0);
5357
5358         /* make sure wptr hasn't changed while processing */
5359         wptr = evergreen_get_ih_wptr(rdev);
5360         if (wptr != rptr)
5361                 goto restart_ih;
5362
5363         return IRQ_HANDLED;
5364 }
5365
5366 static int evergreen_startup(struct radeon_device *rdev)
5367 {
5368         struct radeon_ring *ring;
5369         int r;
5370
5371         /* enable pcie gen2 link */
5372         evergreen_pcie_gen2_enable(rdev);
5373         /* enable aspm */
5374         evergreen_program_aspm(rdev);
5375
5376         /* scratch needs to be initialized before MC */
5377         r = r600_vram_scratch_init(rdev);
5378         if (r)
5379                 return r;
5380
5381         evergreen_mc_program(rdev);
5382
5383         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5384                 r = ni_mc_load_microcode(rdev);
5385                 if (r) {
5386                         DRM_ERROR("Failed to load MC firmware!\n");
5387                         return r;
5388                 }
5389         }
5390
5391         if (rdev->flags & RADEON_IS_AGP) {
5392                 evergreen_agp_enable(rdev);
5393         } else {
5394                 r = evergreen_pcie_gart_enable(rdev);
5395                 if (r)
5396                         return r;
5397         }
5398         evergreen_gpu_init(rdev);
5399
5400         /* allocate rlc buffers */
5401         if (rdev->flags & RADEON_IS_IGP) {
5402                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5403                 rdev->rlc.reg_list_size =
5404                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5405                 rdev->rlc.cs_data = evergreen_cs_data;
5406                 r = sumo_rlc_init(rdev);
5407                 if (r) {
5408                         DRM_ERROR("Failed to init rlc BOs!\n");
5409                         return r;
5410                 }
5411         }
5412
5413         /* allocate wb buffer */
5414         r = radeon_wb_init(rdev);
5415         if (r)
5416                 return r;
5417
5418         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5419         if (r) {
5420                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5421                 return r;
5422         }
5423
5424         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5425         if (r) {
5426                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5427                 return r;
5428         }
5429
5430         r = uvd_v2_2_resume(rdev);
5431         if (!r) {
5432                 r = radeon_fence_driver_start_ring(rdev,
5433                                                    R600_RING_TYPE_UVD_INDEX);
5434                 if (r)
5435                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5436         }
5437
5438         if (r)
5439                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5440
5441         /* Enable IRQ */
5442         if (!rdev->irq.installed) {
5443                 r = radeon_irq_kms_init(rdev);
5444                 if (r)
5445                         return r;
5446         }
5447
5448         r = r600_irq_init(rdev);
5449         if (r) {
5450                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5451                 radeon_irq_kms_fini(rdev);
5452                 return r;
5453         }
5454         evergreen_irq_set(rdev);
5455
5456         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5457         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5458                              RADEON_CP_PACKET2);
5459         if (r)
5460                 return r;
5461
5462         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5463         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5464                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5465         if (r)
5466                 return r;
5467
5468         r = evergreen_cp_load_microcode(rdev);
5469         if (r)
5470                 return r;
5471         r = evergreen_cp_resume(rdev);
5472         if (r)
5473                 return r;
5474         r = r600_dma_resume(rdev);
5475         if (r)
5476                 return r;
5477
5478         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5479         if (ring->ring_size) {
5480                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5481                                      RADEON_CP_PACKET2);
5482                 if (!r)
5483                         r = uvd_v1_0_init(rdev);
5484
5485                 if (r)
5486                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5487         }
5488
5489         r = radeon_ib_pool_init(rdev);
5490         if (r) {
5491                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5492                 return r;
5493         }
5494
5495         r = radeon_audio_init(rdev);
5496         if (r) {
5497                 DRM_ERROR("radeon: audio init failed\n");
5498                 return r;
5499         }
5500
5501         return 0;
5502 }
5503
5504 int evergreen_resume(struct radeon_device *rdev)
5505 {
5506         int r;
5507
5508         /* reset the asic, the gfx blocks are often in a bad state
5509          * after the driver is unloaded or after a resume
5510          */
5511         if (radeon_asic_reset(rdev))
5512                 dev_warn(rdev->dev, "GPU reset failed !\n");
5513         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5514          * posting will perform necessary task to bring back GPU into good
5515          * shape.
5516          */
5517         /* post card */
5518         atom_asic_init(rdev->mode_info.atom_context);
5519
5520         /* init golden registers */
5521         evergreen_init_golden_registers(rdev);
5522
5523         if (rdev->pm.pm_method == PM_METHOD_DPM)
5524                 radeon_pm_resume(rdev);
5525
5526         rdev->accel_working = true;
5527         r = evergreen_startup(rdev);
5528         if (r) {
5529                 DRM_ERROR("evergreen startup failed on resume\n");
5530                 rdev->accel_working = false;
5531                 return r;
5532         }
5533
5534         return r;
5535
5536 }
5537
5538 int evergreen_suspend(struct radeon_device *rdev)
5539 {
5540         radeon_pm_suspend(rdev);
5541         radeon_audio_fini(rdev);
5542         uvd_v1_0_fini(rdev);
5543         radeon_uvd_suspend(rdev);
5544         r700_cp_stop(rdev);
5545         r600_dma_stop(rdev);
5546         evergreen_irq_suspend(rdev);
5547         radeon_wb_disable(rdev);
5548         evergreen_pcie_gart_disable(rdev);
5549
5550         return 0;
5551 }
5552
5553 /* Plan is to move initialization in that function and use
5554  * helper function so that radeon_device_init pretty much
5555  * do nothing more than calling asic specific function. This
5556  * should also allow to remove a bunch of callback function
5557  * like vram_info.
5558  */
5559 int evergreen_init(struct radeon_device *rdev)
5560 {
5561         int r;
5562
5563         /* Read BIOS */
5564         if (!radeon_get_bios(rdev)) {
5565                 if (ASIC_IS_AVIVO(rdev))
5566                         return -EINVAL;
5567         }
5568         /* Must be an ATOMBIOS */
5569         if (!rdev->is_atom_bios) {
5570                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5571                 return -EINVAL;
5572         }
5573         r = radeon_atombios_init(rdev);
5574         if (r)
5575                 return r;
5576         /* reset the asic, the gfx blocks are often in a bad state
5577          * after the driver is unloaded or after a resume
5578          */
5579         if (radeon_asic_reset(rdev))
5580                 dev_warn(rdev->dev, "GPU reset failed !\n");
5581         /* Post card if necessary */
5582         if (!radeon_card_posted(rdev)) {
5583                 if (!rdev->bios) {
5584                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5585                         return -EINVAL;
5586                 }
5587                 DRM_INFO("GPU not posted. posting now...\n");
5588                 atom_asic_init(rdev->mode_info.atom_context);
5589         }
5590         /* init golden registers */
5591         evergreen_init_golden_registers(rdev);
5592         /* Initialize scratch registers */
5593         r600_scratch_init(rdev);
5594         /* Initialize surface registers */
5595         radeon_surface_init(rdev);
5596         /* Initialize clocks */
5597         radeon_get_clock_info(rdev->ddev);
5598         /* Fence driver */
5599         r = radeon_fence_driver_init(rdev);
5600         if (r)
5601                 return r;
5602         /* initialize AGP */
5603         if (rdev->flags & RADEON_IS_AGP) {
5604                 r = radeon_agp_init(rdev);
5605                 if (r)
5606                         radeon_agp_disable(rdev);
5607         }
5608         /* initialize memory controller */
5609         r = evergreen_mc_init(rdev);
5610         if (r)
5611                 return r;
5612         /* Memory manager */
5613         r = radeon_bo_init(rdev);
5614         if (r)
5615                 return r;
5616
5617         if (ASIC_IS_DCE5(rdev)) {
5618                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5619                         r = ni_init_microcode(rdev);
5620                         if (r) {
5621                                 DRM_ERROR("Failed to load firmware!\n");
5622                                 return r;
5623                         }
5624                 }
5625         } else {
5626                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5627                         r = r600_init_microcode(rdev);
5628                         if (r) {
5629                                 DRM_ERROR("Failed to load firmware!\n");
5630                                 return r;
5631                         }
5632                 }
5633         }
5634
5635         /* Initialize power management */
5636         radeon_pm_init(rdev);
5637
5638         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5639         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5640
5641         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5642         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5643
5644         r = radeon_uvd_init(rdev);
5645         if (!r) {
5646                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5647                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5648                                4096);
5649         }
5650
5651         rdev->ih.ring_obj = NULL;
5652         r600_ih_ring_init(rdev, 64 * 1024);
5653
5654         r = r600_pcie_gart_init(rdev);
5655         if (r)
5656                 return r;
5657
5658         rdev->accel_working = true;
5659         r = evergreen_startup(rdev);
5660         if (r) {
5661                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5662                 r700_cp_fini(rdev);
5663                 r600_dma_fini(rdev);
5664                 r600_irq_fini(rdev);
5665                 if (rdev->flags & RADEON_IS_IGP)
5666                         sumo_rlc_fini(rdev);
5667                 radeon_wb_fini(rdev);
5668                 radeon_ib_pool_fini(rdev);
5669                 radeon_irq_kms_fini(rdev);
5670                 evergreen_pcie_gart_fini(rdev);
5671                 rdev->accel_working = false;
5672         }
5673
5674         /* Don't start up if the MC ucode is missing on BTC parts.
5675          * The default clocks and voltages before the MC ucode
5676          * is loaded are not suffient for advanced operations.
5677          */
5678         if (ASIC_IS_DCE5(rdev)) {
5679                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5680                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5681                         return -EINVAL;
5682                 }
5683         }
5684
5685         return 0;
5686 }
5687
5688 void evergreen_fini(struct radeon_device *rdev)
5689 {
5690         radeon_pm_fini(rdev);
5691         radeon_audio_fini(rdev);
5692         r700_cp_fini(rdev);
5693         r600_dma_fini(rdev);
5694         r600_irq_fini(rdev);
5695         if (rdev->flags & RADEON_IS_IGP)
5696                 sumo_rlc_fini(rdev);
5697         radeon_wb_fini(rdev);
5698         radeon_ib_pool_fini(rdev);
5699         radeon_irq_kms_fini(rdev);
5700         uvd_v1_0_fini(rdev);
5701         radeon_uvd_fini(rdev);
5702         evergreen_pcie_gart_fini(rdev);
5703         r600_vram_scratch_fini(rdev);
5704         radeon_gem_fini(rdev);
5705         radeon_fence_driver_fini(rdev);
5706         radeon_agp_fini(rdev);
5707         radeon_bo_fini(rdev);
5708         radeon_atombios_fini(rdev);
5709         kfree(rdev->bios);
5710         rdev->bios = NULL;
5711 }
5712
5713 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5714 {
5715         u32 link_width_cntl, speed_cntl;
5716
5717         if (radeon_pcie_gen2 == 0)
5718                 return;
5719
5720         if (rdev->flags & RADEON_IS_IGP)
5721                 return;
5722
5723         if (!(rdev->flags & RADEON_IS_PCIE))
5724                 return;
5725
5726         /* x2 cards have a special sequence */
5727         if (ASIC_IS_X2(rdev))
5728                 return;
5729
5730         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5731                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5732                 return;
5733
5734         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5735         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5736                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5737                 return;
5738         }
5739
5740         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5741
5742         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5743             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5744
5745                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5746                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5747                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5748
5749                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5750                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5751                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5752
5753                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5754                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5755                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5756
5757                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5758                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5759                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5760
5761                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5762                 speed_cntl |= LC_GEN2_EN_STRAP;
5763                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5764
5765         } else {
5766                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5767                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5768                 if (1)
5769                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5770                 else
5771                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5772                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5773         }
5774 }
5775
5776 void evergreen_program_aspm(struct radeon_device *rdev)
5777 {
5778         u32 data, orig;
5779         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5780         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5781         /* fusion_platform = true
5782          * if the system is a fusion system
5783          * (APU or DGPU in a fusion system).
5784          * todo: check if the system is a fusion platform.
5785          */
5786         bool fusion_platform = false;
5787
5788         if (radeon_aspm == 0)
5789                 return;
5790
5791         if (!(rdev->flags & RADEON_IS_PCIE))
5792                 return;
5793
5794         switch (rdev->family) {
5795         case CHIP_CYPRESS:
5796         case CHIP_HEMLOCK:
5797         case CHIP_JUNIPER:
5798         case CHIP_REDWOOD:
5799         case CHIP_CEDAR:
5800         case CHIP_SUMO:
5801         case CHIP_SUMO2:
5802         case CHIP_PALM:
5803         case CHIP_ARUBA:
5804                 disable_l0s = true;
5805                 break;
5806         default:
5807                 disable_l0s = false;
5808                 break;
5809         }
5810
5811         if (rdev->flags & RADEON_IS_IGP)
5812                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5813
5814         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5815         if (fusion_platform)
5816                 data &= ~MULTI_PIF;
5817         else
5818                 data |= MULTI_PIF;
5819         if (data != orig)
5820                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5821
5822         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5823         if (fusion_platform)
5824                 data &= ~MULTI_PIF;
5825         else
5826                 data |= MULTI_PIF;
5827         if (data != orig)
5828                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5829
5830         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5831         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5832         if (!disable_l0s) {
5833                 if (rdev->family >= CHIP_BARTS)
5834                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5835                 else
5836                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5837         }
5838
5839         if (!disable_l1) {
5840                 if (rdev->family >= CHIP_BARTS)
5841                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5842                 else
5843                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5844
5845                 if (!disable_plloff_in_l1) {
5846                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5847                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5848                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5849                         if (data != orig)
5850                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5851
5852                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5853                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5854                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5855                         if (data != orig)
5856                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5857
5858                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5859                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5860                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5861                         if (data != orig)
5862                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5863
5864                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5865                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5866                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5867                         if (data != orig)
5868                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5869
5870                         if (rdev->family >= CHIP_BARTS) {
5871                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5872                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5873                                 data |= PLL_RAMP_UP_TIME_0(4);
5874                                 if (data != orig)
5875                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5876
5877                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5878                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5879                                 data |= PLL_RAMP_UP_TIME_1(4);
5880                                 if (data != orig)
5881                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5882
5883                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5884                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5885                                 data |= PLL_RAMP_UP_TIME_0(4);
5886                                 if (data != orig)
5887                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5888
5889                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5890                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5891                                 data |= PLL_RAMP_UP_TIME_1(4);
5892                                 if (data != orig)
5893                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5894                         }
5895
5896                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5897                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5898                         data |= LC_DYN_LANES_PWR_STATE(3);
5899                         if (data != orig)
5900                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5901
5902                         if (rdev->family >= CHIP_BARTS) {
5903                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5904                                 data &= ~LS2_EXIT_TIME_MASK;
5905                                 data |= LS2_EXIT_TIME(1);
5906                                 if (data != orig)
5907                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5908
5909                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5910                                 data &= ~LS2_EXIT_TIME_MASK;
5911                                 data |= LS2_EXIT_TIME(1);
5912                                 if (data != orig)
5913                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5914                         }
5915                 }
5916         }
5917
5918         /* evergreen parts only */
5919         if (rdev->family < CHIP_BARTS)
5920                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5921
5922         if (pcie_lc_cntl != pcie_lc_cntl_old)
5923                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5924 }