kernel/drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_trace.h"
  40 #include <drm/drm_atomic.h>
  41 #include <drm/drm_atomic_helper.h>
  42 #include <drm/drm_dp_helper.h>
  43 #include <drm/drm_crtc_helper.h>
  44 #include <drm/drm_plane_helper.h>
  45 #include <drm/drm_rect.h>
  46 #include <linux/dma_remapping.h>
  47
  48 /* Primary plane formats for gen <= 3 */
  49 static const uint32_t i8xx_primary_formats[] = {
  50         DRM_FORMAT_C8,
  51         DRM_FORMAT_RGB565,
  52         DRM_FORMAT_XRGB1555,
  53         DRM_FORMAT_XRGB8888,
  54 };
  55
  56 /* Primary plane formats for gen >= 4 */
  57 static const uint32_t i965_primary_formats[] = {
  58         DRM_FORMAT_C8,
  59         DRM_FORMAT_RGB565,
  60         DRM_FORMAT_XRGB8888,
  61         DRM_FORMAT_XBGR8888,
  62         DRM_FORMAT_XRGB2101010,
  63         DRM_FORMAT_XBGR2101010,
  64 };
  65
  66 static const uint32_t skl_primary_formats[] = {
  67         DRM_FORMAT_C8,
  68         DRM_FORMAT_RGB565,
  69         DRM_FORMAT_XRGB8888,
  70         DRM_FORMAT_XBGR8888,
  71         DRM_FORMAT_ARGB8888,
  72         DRM_FORMAT_ABGR8888,
  73         DRM_FORMAT_XRGB2101010,
  74         DRM_FORMAT_XBGR2101010,
  75         DRM_FORMAT_YUYV,
  76         DRM_FORMAT_YVYU,
  77         DRM_FORMAT_UYVY,
  78         DRM_FORMAT_VYUY,
  79 };
  80
  81 /* Cursor formats */
  82 static const uint32_t intel_cursor_formats[] = {
  83         DRM_FORMAT_ARGB8888,
  84 };
  85
  86 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
  87
  88 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  89                                 struct intel_crtc_state *pipe_config);
  90 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  91                                    struct intel_crtc_state *pipe_config);
  92
  93 static int intel_framebuffer_init(struct drm_device *dev,
  94                                   struct intel_framebuffer *ifb,
  95                                   struct drm_mode_fb_cmd2 *mode_cmd,
  96                                   struct drm_i915_gem_object *obj);
  97 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
  98 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
  99 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 100                                          struct intel_link_m_n *m_n,
 101                                          struct intel_link_m_n *m2_n2);
 102 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 103 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 104 static void intel_set_pipe_csc(struct drm_crtc *crtc);
 105 static void vlv_prepare_pll(struct intel_crtc *crtc,
 106                             const struct intel_crtc_state *pipe_config);
 107 static void chv_prepare_pll(struct intel_crtc *crtc,
 108                             const struct intel_crtc_state *pipe_config);
 109 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 110 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 111 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
 112         struct intel_crtc_state *crtc_state);
 113 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
 114                            int num_connectors);
 115 static void skylake_pfit_enable(struct intel_crtc *crtc);
 116 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 117 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 118 static void intel_modeset_setup_hw_state(struct drm_device *dev);
 119 static void intel_pre_disable_primary(struct drm_crtc *crtc);
 120
 121 typedef struct {
 122         int     min, max;
 123 } intel_range_t;
 124
 125 typedef struct {
 126         int     dot_limit;
 127         int     p2_slow, p2_fast;
 128 } intel_p2_t;
 129
 130 typedef struct intel_limit intel_limit_t;
 131 struct intel_limit {
 132         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
 133         intel_p2_t          p2;
 134 };
 135
 136 /* returns HPLL frequency in kHz */
 137 static int valleyview_get_vco(struct drm_i915_private *dev_priv)
 138 {
 139         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 140
 141         /* Obtain SKU information */
 142         mutex_lock(&dev_priv->sb_lock);
 143         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 144                 CCK_FUSE_HPLL_FREQ_MASK;
 145         mutex_unlock(&dev_priv->sb_lock);
 146
 147         return vco_freq[hpll_freq] * 1000;
 148 }
 149
 150 static int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 151                                   const char *name, u32 reg)
 152 {
 153         u32 val;
 154         int divider;
 155
 156         if (dev_priv->hpll_freq == 0)
 157                 dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
 158
 159         mutex_lock(&dev_priv->sb_lock);
 160         val = vlv_cck_read(dev_priv, reg);
 161         mutex_unlock(&dev_priv->sb_lock);
 162
 163         divider = val & CCK_FREQUENCY_VALUES;
 164
 165         WARN((val & CCK_FREQUENCY_STATUS) !=
 166              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 167              "%s change in progress\n", name);
 168
 169         return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1);
 170 }
 171
 172 int
 173 intel_pch_rawclk(struct drm_device *dev)
 174 {
 175         struct drm_i915_private *dev_priv = dev->dev_private;
 176
 177         WARN_ON(!HAS_PCH_SPLIT(dev));
 178
 179         return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK;
 180 }
 181
 182 /* hrawclock is 1/4 the FSB frequency */
 183 int intel_hrawclk(struct drm_device *dev)
 184 {
 185         struct drm_i915_private *dev_priv = dev->dev_private;
 186         uint32_t clkcfg;
 187
 188         /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
 189         if (IS_VALLEYVIEW(dev))
 190                 return 200;
 191
 192         clkcfg = I915_READ(CLKCFG);
 193         switch (clkcfg & CLKCFG_FSB_MASK) {
 194         case CLKCFG_FSB_400:
 195                 return 100;
 196         case CLKCFG_FSB_533:
 197                 return 133;
 198         case CLKCFG_FSB_667:
 199                 return 166;
 200         case CLKCFG_FSB_800:
 201                 return 200;
 202         case CLKCFG_FSB_1067:
 203                 return 266;
 204         case CLKCFG_FSB_1333:
 205                 return 333;
 206         /* these two are just a guess; one of them might be right */
 207         case CLKCFG_FSB_1600:
 208         case CLKCFG_FSB_1600_ALT:
 209                 return 400;
 210         default:
 211                 return 133;
 212         }
 213 }
 214
 215 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 216 {
 217         if (!IS_VALLEYVIEW(dev_priv))
 218                 return;
 219
 220         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 221                                                       CCK_CZ_CLOCK_CONTROL);
 222
 223         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 224 }
 225
 226 static inline u32 /* units of 100MHz */
 227 intel_fdi_link_freq(struct drm_device *dev)
 228 {
 229         if (IS_GEN5(dev)) {
 230                 struct drm_i915_private *dev_priv = dev->dev_private;
 231                 return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
 232         } else
 233                 return 27;
 234 }
 235
 236 static const intel_limit_t intel_limits_i8xx_dac = {
 237         .dot = { .min = 25000, .max = 350000 },
 238         .vco = { .min = 908000, .max = 1512000 },
 239         .n = { .min = 2, .max = 16 },
 240         .m = { .min = 96, .max = 140 },
 241         .m1 = { .min = 18, .max = 26 },
 242         .m2 = { .min = 6, .max = 16 },
 243         .p = { .min = 4, .max = 128 },
 244         .p1 = { .min = 2, .max = 33 },
 245         .p2 = { .dot_limit = 165000,
 246                 .p2_slow = 4, .p2_fast = 2 },
 247 };
 248
 249 static const intel_limit_t intel_limits_i8xx_dvo = {
 250         .dot = { .min = 25000, .max = 350000 },
 251         .vco = { .min = 908000, .max = 1512000 },
 252         .n = { .min = 2, .max = 16 },
 253         .m = { .min = 96, .max = 140 },
 254         .m1 = { .min = 18, .max = 26 },
 255         .m2 = { .min = 6, .max = 16 },
 256         .p = { .min = 4, .max = 128 },
 257         .p1 = { .min = 2, .max = 33 },
 258         .p2 = { .dot_limit = 165000,
 259                 .p2_slow = 4, .p2_fast = 4 },
 260 };
 261
 262 static const intel_limit_t intel_limits_i8xx_lvds = {
 263         .dot = { .min = 25000, .max = 350000 },
 264         .vco = { .min = 908000, .max = 1512000 },
 265         .n = { .min = 2, .max = 16 },
 266         .m = { .min = 96, .max = 140 },
 267         .m1 = { .min = 18, .max = 26 },
 268         .m2 = { .min = 6, .max = 16 },
 269         .p = { .min = 4, .max = 128 },
 270         .p1 = { .min = 1, .max = 6 },
 271         .p2 = { .dot_limit = 165000,
 272                 .p2_slow = 14, .p2_fast = 7 },
 273 };
 274
 275 static const intel_limit_t intel_limits_i9xx_sdvo = {
 276         .dot = { .min = 20000, .max = 400000 },
 277         .vco = { .min = 1400000, .max = 2800000 },
 278         .n = { .min = 1, .max = 6 },
 279         .m = { .min = 70, .max = 120 },
 280         .m1 = { .min = 8, .max = 18 },
 281         .m2 = { .min = 3, .max = 7 },
 282         .p = { .min = 5, .max = 80 },
 283         .p1 = { .min = 1, .max = 8 },
 284         .p2 = { .dot_limit = 200000,
 285                 .p2_slow = 10, .p2_fast = 5 },
 286 };
 287
 288 static const intel_limit_t intel_limits_i9xx_lvds = {
 289         .dot = { .min = 20000, .max = 400000 },
 290         .vco = { .min = 1400000, .max = 2800000 },
 291         .n = { .min = 1, .max = 6 },
 292         .m = { .min = 70, .max = 120 },
 293         .m1 = { .min = 8, .max = 18 },
 294         .m2 = { .min = 3, .max = 7 },
 295         .p = { .min = 7, .max = 98 },
 296         .p1 = { .min = 1, .max = 8 },
 297         .p2 = { .dot_limit = 112000,
 298                 .p2_slow = 14, .p2_fast = 7 },
 299 };
 300
 301
 302 static const intel_limit_t intel_limits_g4x_sdvo = {
 303         .dot = { .min = 25000, .max = 270000 },
 304         .vco = { .min = 1750000, .max = 3500000},
 305         .n = { .min = 1, .max = 4 },
 306         .m = { .min = 104, .max = 138 },
 307         .m1 = { .min = 17, .max = 23 },
 308         .m2 = { .min = 5, .max = 11 },
 309         .p = { .min = 10, .max = 30 },
 310         .p1 = { .min = 1, .max = 3},
 311         .p2 = { .dot_limit = 270000,
 312                 .p2_slow = 10,
 313                 .p2_fast = 10
 314         },
 315 };
 316
 317 static const intel_limit_t intel_limits_g4x_hdmi = {
 318         .dot = { .min = 22000, .max = 400000 },
 319         .vco = { .min = 1750000, .max = 3500000},
 320         .n = { .min = 1, .max = 4 },
 321         .m = { .min = 104, .max = 138 },
 322         .m1 = { .min = 16, .max = 23 },
 323         .m2 = { .min = 5, .max = 11 },
 324         .p = { .min = 5, .max = 80 },
 325         .p1 = { .min = 1, .max = 8},
 326         .p2 = { .dot_limit = 165000,
 327                 .p2_slow = 10, .p2_fast = 5 },
 328 };
 329
 330 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
 331         .dot = { .min = 20000, .max = 115000 },
 332         .vco = { .min = 1750000, .max = 3500000 },
 333         .n = { .min = 1, .max = 3 },
 334         .m = { .min = 104, .max = 138 },
 335         .m1 = { .min = 17, .max = 23 },
 336         .m2 = { .min = 5, .max = 11 },
 337         .p = { .min = 28, .max = 112 },
 338         .p1 = { .min = 2, .max = 8 },
 339         .p2 = { .dot_limit = 0,
 340                 .p2_slow = 14, .p2_fast = 14
 341         },
 342 };
 343
 344 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
 345         .dot = { .min = 80000, .max = 224000 },
 346         .vco = { .min = 1750000, .max = 3500000 },
 347         .n = { .min = 1, .max = 3 },
 348         .m = { .min = 104, .max = 138 },
 349         .m1 = { .min = 17, .max = 23 },
 350         .m2 = { .min = 5, .max = 11 },
 351         .p = { .min = 14, .max = 42 },
 352         .p1 = { .min = 2, .max = 6 },
 353         .p2 = { .dot_limit = 0,
 354                 .p2_slow = 7, .p2_fast = 7
 355         },
 356 };
 357
 358 static const intel_limit_t intel_limits_pineview_sdvo = {
 359         .dot = { .min = 20000, .max = 400000},
 360         .vco = { .min = 1700000, .max = 3500000 },
 361         /* Pineview's Ncounter is a ring counter */
 362         .n = { .min = 3, .max = 6 },
 363         .m = { .min = 2, .max = 256 },
 364         /* Pineview only has one combined m divider, which we treat as m2. */
 365         .m1 = { .min = 0, .max = 0 },
 366         .m2 = { .min = 0, .max = 254 },
 367         .p = { .min = 5, .max = 80 },
 368         .p1 = { .min = 1, .max = 8 },
 369         .p2 = { .dot_limit = 200000,
 370                 .p2_slow = 10, .p2_fast = 5 },
 371 };
 372
 373 static const intel_limit_t intel_limits_pineview_lvds = {
 374         .dot = { .min = 20000, .max = 400000 },
 375         .vco = { .min = 1700000, .max = 3500000 },
 376         .n = { .min = 3, .max = 6 },
 377         .m = { .min = 2, .max = 256 },
 378         .m1 = { .min = 0, .max = 0 },
 379         .m2 = { .min = 0, .max = 254 },
 380         .p = { .min = 7, .max = 112 },
 381         .p1 = { .min = 1, .max = 8 },
 382         .p2 = { .dot_limit = 112000,
 383                 .p2_slow = 14, .p2_fast = 14 },
 384 };
 385
 386 /* Ironlake / Sandybridge
 387  *
 388  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 389  * the range value for them is (actual_value - 2).
 390  */
 391 static const intel_limit_t intel_limits_ironlake_dac = {
 392         .dot = { .min = 25000, .max = 350000 },
 393         .vco = { .min = 1760000, .max = 3510000 },
 394         .n = { .min = 1, .max = 5 },
 395         .m = { .min = 79, .max = 127 },
 396         .m1 = { .min = 12, .max = 22 },
 397         .m2 = { .min = 5, .max = 9 },
 398         .p = { .min = 5, .max = 80 },
 399         .p1 = { .min = 1, .max = 8 },
 400         .p2 = { .dot_limit = 225000,
 401                 .p2_slow = 10, .p2_fast = 5 },
 402 };
 403
 404 static const intel_limit_t intel_limits_ironlake_single_lvds = {
 405         .dot = { .min = 25000, .max = 350000 },
 406         .vco = { .min = 1760000, .max = 3510000 },
 407         .n = { .min = 1, .max = 3 },
 408         .m = { .min = 79, .max = 118 },
 409         .m1 = { .min = 12, .max = 22 },
 410         .m2 = { .min = 5, .max = 9 },
 411         .p = { .min = 28, .max = 112 },
 412         .p1 = { .min = 2, .max = 8 },
 413         .p2 = { .dot_limit = 225000,
 414                 .p2_slow = 14, .p2_fast = 14 },
 415 };
 416
 417 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 418         .dot = { .min = 25000, .max = 350000 },
 419         .vco = { .min = 1760000, .max = 3510000 },
 420         .n = { .min = 1, .max = 3 },
 421         .m = { .min = 79, .max = 127 },
 422         .m1 = { .min = 12, .max = 22 },
 423         .m2 = { .min = 5, .max = 9 },
 424         .p = { .min = 14, .max = 56 },
 425         .p1 = { .min = 2, .max = 8 },
 426         .p2 = { .dot_limit = 225000,
 427                 .p2_slow = 7, .p2_fast = 7 },
 428 };
 429
 430 /* LVDS 100mhz refclk limits. */
 431 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
 432         .dot = { .min = 25000, .max = 350000 },
 433         .vco = { .min = 1760000, .max = 3510000 },
 434         .n = { .min = 1, .max = 2 },
 435         .m = { .min = 79, .max = 126 },
 436         .m1 = { .min = 12, .max = 22 },
 437         .m2 = { .min = 5, .max = 9 },
 438         .p = { .min = 28, .max = 112 },
 439         .p1 = { .min = 2, .max = 8 },
 440         .p2 = { .dot_limit = 225000,
 441                 .p2_slow = 14, .p2_fast = 14 },
 442 };
 443
 444 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
 445         .dot = { .min = 25000, .max = 350000 },
 446         .vco = { .min = 1760000, .max = 3510000 },
 447         .n = { .min = 1, .max = 3 },
 448         .m = { .min = 79, .max = 126 },
 449         .m1 = { .min = 12, .max = 22 },
 450         .m2 = { .min = 5, .max = 9 },
 451         .p = { .min = 14, .max = 42 },
 452         .p1 = { .min = 2, .max = 6 },
 453         .p2 = { .dot_limit = 225000,
 454                 .p2_slow = 7, .p2_fast = 7 },
 455 };
 456
 457 static const intel_limit_t intel_limits_vlv = {
 458          /*
 459           * These are the data rate limits (measured in fast clocks)
 460           * since those are the strictest limits we have. The fast
 461           * clock and actual rate limits are more relaxed, so checking
 462           * them would make no difference.
 463           */
 464         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 465         .vco = { .min = 4000000, .max = 6000000 },
 466         .n = { .min = 1, .max = 7 },
 467         .m1 = { .min = 2, .max = 3 },
 468         .m2 = { .min = 11, .max = 156 },
 469         .p1 = { .min = 2, .max = 3 },
 470         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 471 };
 472
 473 static const intel_limit_t intel_limits_chv = {
 474         /*
 475          * These are the data rate limits (measured in fast clocks)
 476          * since those are the strictest limits we have.  The fast
 477          * clock and actual rate limits are more relaxed, so checking
 478          * them would make no difference.
 479          */
 480         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 481         .vco = { .min = 4800000, .max = 6480000 },
 482         .n = { .min = 1, .max = 1 },
 483         .m1 = { .min = 2, .max = 2 },
 484         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 485         .p1 = { .min = 2, .max = 4 },
 486         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 487 };
 488
 489 static const intel_limit_t intel_limits_bxt = {
 490         /* FIXME: find real dot limits */
 491         .dot = { .min = 0, .max = INT_MAX },
 492         .vco = { .min = 4800000, .max = 6700000 },
 493         .n = { .min = 1, .max = 1 },
 494         .m1 = { .min = 2, .max = 2 },
 495         /* FIXME: find real m2 limits */
 496         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 497         .p1 = { .min = 2, .max = 4 },
 498         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 499 };
 500
 501 static bool
 502 needs_modeset(struct drm_crtc_state *state)
 503 {
 504         return drm_atomic_crtc_needs_modeset(state);
 505 }
 506
 507 /**
 508  * Returns whether any output on the specified pipe is of the specified type
 509  */
 510 bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type)
 511 {
 512         struct drm_device *dev = crtc->base.dev;
 513         struct intel_encoder *encoder;
 514
 515         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
 516                 if (encoder->type == type)
 517                         return true;
 518
 519         return false;
 520 }
 521
 522 /**
 523  * Returns whether any output on the specified pipe will have the specified
 524  * type after a staged modeset is complete, i.e., the same as
 525  * intel_pipe_has_type() but looking at encoder->new_crtc instead of
 526  * encoder->crtc.
 527  */
 528 static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
 529                                       int type)
 530 {
 531         struct drm_atomic_state *state = crtc_state->base.state;
 532         struct drm_connector *connector;
 533         struct drm_connector_state *connector_state;
 534         struct intel_encoder *encoder;
 535         int i, num_connectors = 0;
 536
 537         for_each_connector_in_state(state, connector, connector_state, i) {
 538                 if (connector_state->crtc != crtc_state->base.crtc)
 539                         continue;
 540
 541                 num_connectors++;
 542
 543                 encoder = to_intel_encoder(connector_state->best_encoder);
 544                 if (encoder->type == type)
 545                         return true;
 546         }
 547
 548         WARN_ON(num_connectors == 0);
 549
 550         return false;
 551 }
 552
 553 static const intel_limit_t *
 554 intel_ironlake_limit(struct intel_crtc_state *crtc_state, int refclk)
 555 {
 556         struct drm_device *dev = crtc_state->base.crtc->dev;
 557         const intel_limit_t *limit;
 558
 559         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 560                 if (intel_is_dual_link_lvds(dev)) {
 561                         if (refclk == 100000)
 562                                 limit = &intel_limits_ironlake_dual_lvds_100m;
 563                         else
 564                                 limit = &intel_limits_ironlake_dual_lvds;
 565                 } else {
 566                         if (refclk == 100000)
 567                                 limit = &intel_limits_ironlake_single_lvds_100m;
 568                         else
 569                                 limit = &intel_limits_ironlake_single_lvds;
 570                 }
 571         } else
 572                 limit = &intel_limits_ironlake_dac;
 573
 574         return limit;
 575 }
 576
 577 static const intel_limit_t *
 578 intel_g4x_limit(struct intel_crtc_state *crtc_state)
 579 {
 580         struct drm_device *dev = crtc_state->base.crtc->dev;
 581         const intel_limit_t *limit;
 582
 583         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 584                 if (intel_is_dual_link_lvds(dev))
 585                         limit = &intel_limits_g4x_dual_channel_lvds;
 586                 else
 587                         limit = &intel_limits_g4x_single_channel_lvds;
 588         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) ||
 589                    intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 590                 limit = &intel_limits_g4x_hdmi;
 591         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) {
 592                 limit = &intel_limits_g4x_sdvo;
 593         } else /* The option is for other outputs */
 594                 limit = &intel_limits_i9xx_sdvo;
 595
 596         return limit;
 597 }
 598
 599 static const intel_limit_t *
 600 intel_limit(struct intel_crtc_state *crtc_state, int refclk)
 601 {
 602         struct drm_device *dev = crtc_state->base.crtc->dev;
 603         const intel_limit_t *limit;
 604
 605         if (IS_BROXTON(dev))
 606                 limit = &intel_limits_bxt;
 607         else if (HAS_PCH_SPLIT(dev))
 608                 limit = intel_ironlake_limit(crtc_state, refclk);
 609         else if (IS_G4X(dev)) {
 610                 limit = intel_g4x_limit(crtc_state);
 611         } else if (IS_PINEVIEW(dev)) {
 612                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 613                         limit = &intel_limits_pineview_lvds;
 614                 else
 615                         limit = &intel_limits_pineview_sdvo;
 616         } else if (IS_CHERRYVIEW(dev)) {
 617                 limit = &intel_limits_chv;
 618         } else if (IS_VALLEYVIEW(dev)) {
 619                 limit = &intel_limits_vlv;
 620         } else if (!IS_GEN2(dev)) {
 621                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 622                         limit = &intel_limits_i9xx_lvds;
 623                 else
 624                         limit = &intel_limits_i9xx_sdvo;
 625         } else {
 626                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 627                         limit = &intel_limits_i8xx_lvds;
 628                 else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
 629                         limit = &intel_limits_i8xx_dvo;
 630                 else
 631                         limit = &intel_limits_i8xx_dac;
 632         }
 633         return limit;
 634 }
 635
 636 /*
 637  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 638  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 639  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 640  * The helpers' return value is the rate of the clock that is fed to the
 641  * display engine's pipe which can be the above fast dot clock rate or a
 642  * divided-down version of it.
 643  */
 644 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 645 static int pnv_calc_dpll_params(int refclk, intel_clock_t *clock)
 646 {
 647         clock->m = clock->m2 + 2;
 648         clock->p = clock->p1 * clock->p2;
 649         if (WARN_ON(clock->n == 0 || clock->p == 0))
 650                 return 0;
 651         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 652         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 653
 654         return clock->dot;
 655 }
 656
 657 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 658 {
 659         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 660 }
 661
 662 static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock)
 663 {
 664         clock->m = i9xx_dpll_compute_m(clock);
 665         clock->p = clock->p1 * clock->p2;
 666         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 667                 return 0;
 668         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 669         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 670
 671         return clock->dot;
 672 }
 673
 674 static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock)
 675 {
 676         clock->m = clock->m1 * clock->m2;
 677         clock->p = clock->p1 * clock->p2;
 678         if (WARN_ON(clock->n == 0 || clock->p == 0))
 679                 return 0;
 680         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 681         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 682
 683         return clock->dot / 5;
 684 }
 685
 686 int chv_calc_dpll_params(int refclk, intel_clock_t *clock)
 687 {
 688         clock->m = clock->m1 * clock->m2;
 689         clock->p = clock->p1 * clock->p2;
 690         if (WARN_ON(clock->n == 0 || clock->p == 0))
 691                 return 0;
 692         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 693                         clock->n << 22);
 694         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 695
 696         return clock->dot / 5;
 697 }
 698
 699 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 700 /**
 701  * Returns whether the given set of divisors are valid for a given refclk with
 702  * the given connectors.
 703  */
 704
 705 static bool intel_PLL_is_valid(struct drm_device *dev,
 706                                const intel_limit_t *limit,
 707                                const intel_clock_t *clock)
 708 {
 709         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 710                 INTELPllInvalid("n out of range\n");
 711         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 712                 INTELPllInvalid("p1 out of range\n");
 713         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 714                 INTELPllInvalid("m2 out of range\n");
 715         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 716                 INTELPllInvalid("m1 out of range\n");
 717
 718         if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
 719                 if (clock->m1 <= clock->m2)
 720                         INTELPllInvalid("m1 <= m2\n");
 721
 722         if (!IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) {
 723                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 724                         INTELPllInvalid("p out of range\n");
 725                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 726                         INTELPllInvalid("m out of range\n");
 727         }
 728
 729         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 730                 INTELPllInvalid("vco out of range\n");
 731         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 732          * connector, etc., rather than just a single range.
 733          */
 734         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 735                 INTELPllInvalid("dot out of range\n");
 736
 737         return true;
 738 }
 739
 740 static int
 741 i9xx_select_p2_div(const intel_limit_t *limit,
 742                    const struct intel_crtc_state *crtc_state,
 743                    int target)
 744 {
 745         struct drm_device *dev = crtc_state->base.crtc->dev;
 746
 747         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 748                 /*
 749                  * For LVDS just rely on its current settings for dual-channel.
 750                  * We haven't figured out how to reliably set up different
 751                  * single/dual channel state, if we even can.
 752                  */
 753                 if (intel_is_dual_link_lvds(dev))
 754                         return limit->p2.p2_fast;
 755                 else
 756                         return limit->p2.p2_slow;
 757         } else {
 758                 if (target < limit->p2.dot_limit)
 759                         return limit->p2.p2_slow;
 760                 else
 761                         return limit->p2.p2_fast;
 762         }
 763 }
 764
 765 static bool
 766 i9xx_find_best_dpll(const intel_limit_t *limit,
 767                     struct intel_crtc_state *crtc_state,
 768                     int target, int refclk, intel_clock_t *match_clock,
 769                     intel_clock_t *best_clock)
 770 {
 771         struct drm_device *dev = crtc_state->base.crtc->dev;
 772         intel_clock_t clock;
 773         int err = target;
 774
 775         memset(best_clock, 0, sizeof(*best_clock));
 776
 777         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 778
 779         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 780              clock.m1++) {
 781                 for (clock.m2 = limit->m2.min;
 782                      clock.m2 <= limit->m2.max; clock.m2++) {
 783                         if (clock.m2 >= clock.m1)
 784                                 break;
 785                         for (clock.n = limit->n.min;
 786                              clock.n <= limit->n.max; clock.n++) {
 787                                 for (clock.p1 = limit->p1.min;
 788                                         clock.p1 <= limit->p1.max; clock.p1++) {
 789                                         int this_err;
 790
 791                                         i9xx_calc_dpll_params(refclk, &clock);
 792                                         if (!intel_PLL_is_valid(dev, limit,
 793                                                                 &clock))
 794                                                 continue;
 795                                         if (match_clock &&
 796                                             clock.p != match_clock->p)
 797                                                 continue;
 798
 799                                         this_err = abs(clock.dot - target);
 800                                         if (this_err < err) {
 801                                                 *best_clock = clock;
 802                                                 err = this_err;
 803                                         }
 804                                 }
 805                         }
 806                 }
 807         }
 808
 809         return (err != target);
 810 }
 811
 812 static bool
 813 pnv_find_best_dpll(const intel_limit_t *limit,
 814                    struct intel_crtc_state *crtc_state,
 815                    int target, int refclk, intel_clock_t *match_clock,
 816                    intel_clock_t *best_clock)
 817 {
 818         struct drm_device *dev = crtc_state->base.crtc->dev;
 819         intel_clock_t clock;
 820         int err = target;
 821
 822         memset(best_clock, 0, sizeof(*best_clock));
 823
 824         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 825
 826         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 827              clock.m1++) {
 828                 for (clock.m2 = limit->m2.min;
 829                      clock.m2 <= limit->m2.max; clock.m2++) {
 830                         for (clock.n = limit->n.min;
 831                              clock.n <= limit->n.max; clock.n++) {
 832                                 for (clock.p1 = limit->p1.min;
 833                                         clock.p1 <= limit->p1.max; clock.p1++) {
 834                                         int this_err;
 835
 836                                         pnv_calc_dpll_params(refclk, &clock);
 837                                         if (!intel_PLL_is_valid(dev, limit,
 838                                                                 &clock))
 839                                                 continue;
 840                                         if (match_clock &&
 841                                             clock.p != match_clock->p)
 842                                                 continue;
 843
 844                                         this_err = abs(clock.dot - target);
 845                                         if (this_err < err) {
 846                                                 *best_clock = clock;
 847                                                 err = this_err;
 848                                         }
 849                                 }
 850                         }
 851                 }
 852         }
 853
 854         return (err != target);
 855 }
 856
 857 static bool
 858 g4x_find_best_dpll(const intel_limit_t *limit,
 859                    struct intel_crtc_state *crtc_state,
 860                    int target, int refclk, intel_clock_t *match_clock,
 861                    intel_clock_t *best_clock)
 862 {
 863         struct drm_device *dev = crtc_state->base.crtc->dev;
 864         intel_clock_t clock;
 865         int max_n;
 866         bool found = false;
 867         /* approximately equals target * 0.00585 */
 868         int err_most = (target >> 8) + (target >> 9);
 869
 870         memset(best_clock, 0, sizeof(*best_clock));
 871
 872         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 873
 874         max_n = limit->n.max;
 875         /* based on hardware requirement, prefer smaller n to precision */
 876         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 877                 /* based on hardware requirement, prefere larger m1,m2 */
 878                 for (clock.m1 = limit->m1.max;
 879                      clock.m1 >= limit->m1.min; clock.m1--) {
 880                         for (clock.m2 = limit->m2.max;
 881                              clock.m2 >= limit->m2.min; clock.m2--) {
 882                                 for (clock.p1 = limit->p1.max;
 883                                      clock.p1 >= limit->p1.min; clock.p1--) {
 884                                         int this_err;
 885
 886                                         i9xx_calc_dpll_params(refclk, &clock);
 887                                         if (!intel_PLL_is_valid(dev, limit,
 888                                                                 &clock))
 889                                                 continue;
 890
 891                                         this_err = abs(clock.dot - target);
 892                                         if (this_err < err_most) {
 893                                                 *best_clock = clock;
 894                                                 err_most = this_err;
 895                                                 max_n = clock.n;
 896                                                 found = true;
 897                                         }
 898                                 }
 899                         }
 900                 }
 901         }
 902         return found;
 903 }
 904
 905 /*
 906  * Check if the calculated PLL configuration is more optimal compared to the
 907  * best configuration and error found so far. Return the calculated error.
 908  */
 909 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 910                                const intel_clock_t *calculated_clock,
 911                                const intel_clock_t *best_clock,
 912                                unsigned int best_error_ppm,
 913                                unsigned int *error_ppm)
 914 {
 915         /*
 916          * For CHV ignore the error and consider only the P value.
 917          * Prefer a bigger P value based on HW requirements.
 918          */
 919         if (IS_CHERRYVIEW(dev)) {
 920                 *error_ppm = 0;
 921
 922                 return calculated_clock->p > best_clock->p;
 923         }
 924
 925         if (WARN_ON_ONCE(!target_freq))
 926                 return false;
 927
 928         *error_ppm = div_u64(1000000ULL *
 929                                 abs(target_freq - calculated_clock->dot),
 930                              target_freq);
 931         /*
 932          * Prefer a better P value over a better (smaller) error if the error
 933          * is small. Ensure this preference for future configurations too by
 934          * setting the error to 0.
 935          */
 936         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 937                 *error_ppm = 0;
 938
 939                 return true;
 940         }
 941
 942         return *error_ppm + 10 < best_error_ppm;
 943 }
 944
 945 static bool
 946 vlv_find_best_dpll(const intel_limit_t *limit,
 947                    struct intel_crtc_state *crtc_state,
 948                    int target, int refclk, intel_clock_t *match_clock,
 949                    intel_clock_t *best_clock)
 950 {
 951         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 952         struct drm_device *dev = crtc->base.dev;
 953         intel_clock_t clock;
 954         unsigned int bestppm = 1000000;
 955         /* min update 19.2 MHz */
 956         int max_n = min(limit->n.max, refclk / 19200);
 957         bool found = false;
 958
 959         target *= 5; /* fast clock */
 960
 961         memset(best_clock, 0, sizeof(*best_clock));
 962
 963         /* based on hardware requirement, prefer smaller n to precision */
 964         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 965                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 966                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 967                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 968                                 clock.p = clock.p1 * clock.p2;
 969                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 970                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 971                                         unsigned int ppm;
 972
 973                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 974                                                                      refclk * clock.m1);
 975
 976                                         vlv_calc_dpll_params(refclk, &clock);
 977
 978                                         if (!intel_PLL_is_valid(dev, limit,
 979                                                                 &clock))
 980                                                 continue;
 981
 982                                         if (!vlv_PLL_is_optimal(dev, target,
 983                                                                 &clock,
 984                                                                 best_clock,
 985                                                                 bestppm, &ppm))
 986                                                 continue;
 987
 988                                         *best_clock = clock;
 989                                         bestppm = ppm;
 990                                         found = true;
 991                                 }
 992                         }
 993                 }
 994         }
 995
 996         return found;
 997 }
 998
 999 static bool
1000 chv_find_best_dpll(const intel_limit_t *limit,
1001                    struct intel_crtc_state *crtc_state,
1002                    int target, int refclk, intel_clock_t *match_clock,
1003                    intel_clock_t *best_clock)
1004 {
1005         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1006         struct drm_device *dev = crtc->base.dev;
1007         unsigned int best_error_ppm;
1008         intel_clock_t clock;
1009         uint64_t m2;
1010         int found = false;
1011
1012         memset(best_clock, 0, sizeof(*best_clock));
1013         best_error_ppm = 1000000;
1014
1015         /*
1016          * Based on hardware doc, the n always set to 1, and m1 always
1017          * set to 2.  If requires to support 200Mhz refclk, we need to
1018          * revisit this because n may not 1 anymore.
1019          */
1020         clock.n = 1, clock.m1 = 2;
1021         target *= 5;    /* fast clock */
1022
1023         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1024                 for (clock.p2 = limit->p2.p2_fast;
1025                                 clock.p2 >= limit->p2.p2_slow;
1026                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1027                         unsigned int error_ppm;
1028
1029                         clock.p = clock.p1 * clock.p2;
1030
1031                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
1032                                         clock.n) << 22, refclk * clock.m1);
1033
1034                         if (m2 > INT_MAX/clock.m1)
1035                                 continue;
1036
1037                         clock.m2 = m2;
1038
1039                         chv_calc_dpll_params(refclk, &clock);
1040
1041                         if (!intel_PLL_is_valid(dev, limit, &clock))
1042                                 continue;
1043
1044                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1045                                                 best_error_ppm, &error_ppm))
1046                                 continue;
1047
1048                         *best_clock = clock;
1049                         best_error_ppm = error_ppm;
1050                         found = true;
1051                 }
1052         }
1053
1054         return found;
1055 }
1056
1057 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
1058                         intel_clock_t *best_clock)
1059 {
1060         int refclk = i9xx_get_refclk(crtc_state, 0);
1061
1062         return chv_find_best_dpll(intel_limit(crtc_state, refclk), crtc_state,
1063                                   target_clock, refclk, NULL, best_clock);
1064 }
1065
1066 bool intel_crtc_active(struct drm_crtc *crtc)
1067 {
1068         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1069
1070         /* Be paranoid as we can arrive here with only partial
1071          * state retrieved from the hardware during setup.
1072          *
1073          * We can ditch the adjusted_mode.crtc_clock check as soon
1074          * as Haswell has gained clock readout/fastboot support.
1075          *
1076          * We can ditch the crtc->primary->fb check as soon as we can
1077          * properly reconstruct framebuffers.
1078          *
1079          * FIXME: The intel_crtc->active here should be switched to
1080          * crtc->state->active once we have proper CRTC states wired up
1081          * for atomic.
1082          */
1083         return intel_crtc->active && crtc->primary->state->fb &&
1084                 intel_crtc->config->base.adjusted_mode.crtc_clock;
1085 }
1086
1087 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1088                                              enum pipe pipe)
1089 {
1090         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1091         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1092
1093         return intel_crtc->config->cpu_transcoder;
1094 }
1095
1096 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
1097 {
1098         struct drm_i915_private *dev_priv = dev->dev_private;
1099         u32 reg = PIPEDSL(pipe);
1100         u32 line1, line2;
1101         u32 line_mask;
1102
1103         if (IS_GEN2(dev))
1104                 line_mask = DSL_LINEMASK_GEN2;
1105         else
1106                 line_mask = DSL_LINEMASK_GEN3;
1107
1108         line1 = I915_READ(reg) & line_mask;
1109         msleep(5);
1110         line2 = I915_READ(reg) & line_mask;
1111
1112         return line1 == line2;
1113 }
1114
1115 /*
1116  * intel_wait_for_pipe_off - wait for pipe to turn off
1117  * @crtc: crtc whose pipe to wait for
1118  *
1119  * After disabling a pipe, we can't wait for vblank in the usual way,
1120  * spinning on the vblank interrupt status bit, since we won't actually
1121  * see an interrupt when the pipe is disabled.
1122  *
1123  * On Gen4 and above:
1124  *   wait for the pipe register state bit to turn off
1125  *
1126  * Otherwise:
1127  *   wait for the display line value to settle (it usually
1128  *   ends up stopping at the start of the next frame).
1129  *
1130  */
1131 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1132 {
1133         struct drm_device *dev = crtc->base.dev;
1134         struct drm_i915_private *dev_priv = dev->dev_private;
1135         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1136         enum pipe pipe = crtc->pipe;
1137
1138         if (INTEL_INFO(dev)->gen >= 4) {
1139                 int reg = PIPECONF(cpu_transcoder);
1140
1141                 /* Wait for the Pipe State to go off */
1142                 if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
1143                              100))
1144                         WARN(1, "pipe_off wait timed out\n");
1145         } else {
1146                 /* Wait for the display line to settle */
1147                 if (wait_for(pipe_dsl_stopped(dev, pipe), 100))
1148                         WARN(1, "pipe_off wait timed out\n");
1149         }
1150 }
1151
1152 static const char *state_string(bool enabled)
1153 {
1154         return enabled ? "on" : "off";
1155 }
1156
1157 /* Only for pre-ILK configs */
1158 void assert_pll(struct drm_i915_private *dev_priv,
1159                 enum pipe pipe, bool state)
1160 {
1161         u32 val;
1162         bool cur_state;
1163
1164         val = I915_READ(DPLL(pipe));
1165         cur_state = !!(val & DPLL_VCO_ENABLE);
1166         I915_STATE_WARN(cur_state != state,
1167              "PLL state assertion failure (expected %s, current %s)\n",
1168              state_string(state), state_string(cur_state));
1169 }
1170
1171 /* XXX: the dsi pll is shared between MIPI DSI ports */
1172 static void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1173 {
1174         u32 val;
1175         bool cur_state;
1176
1177         mutex_lock(&dev_priv->sb_lock);
1178         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1179         mutex_unlock(&dev_priv->sb_lock);
1180
1181         cur_state = val & DSI_PLL_VCO_EN;
1182         I915_STATE_WARN(cur_state != state,
1183              "DSI PLL state assertion failure (expected %s, current %s)\n",
1184              state_string(state), state_string(cur_state));
1185 }
1186 #define assert_dsi_pll_enabled(d) assert_dsi_pll(d, true)
1187 #define assert_dsi_pll_disabled(d) assert_dsi_pll(d, false)
1188
1189 struct intel_shared_dpll *
1190 intel_crtc_to_shared_dpll(struct intel_crtc *crtc)
1191 {
1192         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
1193
1194         if (crtc->config->shared_dpll < 0)
1195                 return NULL;
1196
1197         return &dev_priv->shared_dplls[crtc->config->shared_dpll];
1198 }
1199
1200 /* For ILK+ */
1201 void assert_shared_dpll(struct drm_i915_private *dev_priv,
1202                         struct intel_shared_dpll *pll,
1203                         bool state)
1204 {
1205         bool cur_state;
1206         struct intel_dpll_hw_state hw_state;
1207
1208         if (WARN (!pll,
1209                   "asserting DPLL %s with no DPLL\n", state_string(state)))
1210                 return;
1211
1212         cur_state = pll->get_hw_state(dev_priv, pll, &hw_state);
1213         I915_STATE_WARN(cur_state != state,
1214              "%s assertion failure (expected %s, current %s)\n",
1215              pll->name, state_string(state), state_string(cur_state));
1216 }
1217
1218 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1219                           enum pipe pipe, bool state)
1220 {
1221         bool cur_state;
1222         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1223                                                                       pipe);
1224
1225         if (HAS_DDI(dev_priv->dev)) {
1226                 /* DDI does not have a specific FDI_TX register */
1227                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1228                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1229         } else {
1230                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1231                 cur_state = !!(val & FDI_TX_ENABLE);
1232         }
1233         I915_STATE_WARN(cur_state != state,
1234              "FDI TX state assertion failure (expected %s, current %s)\n",
1235              state_string(state), state_string(cur_state));
1236 }
1237 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1238 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1239
1240 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1241                           enum pipe pipe, bool state)
1242 {
1243         u32 val;
1244         bool cur_state;
1245
1246         val = I915_READ(FDI_RX_CTL(pipe));
1247         cur_state = !!(val & FDI_RX_ENABLE);
1248         I915_STATE_WARN(cur_state != state,
1249              "FDI RX state assertion failure (expected %s, current %s)\n",
1250              state_string(state), state_string(cur_state));
1251 }
1252 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1253 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1254
1255 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1256                                       enum pipe pipe)
1257 {
1258         u32 val;
1259
1260         /* ILK FDI PLL is always enabled */
1261         if (INTEL_INFO(dev_priv->dev)->gen == 5)
1262                 return;
1263
1264         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1265         if (HAS_DDI(dev_priv->dev))
1266                 return;
1267
1268         val = I915_READ(FDI_TX_CTL(pipe));
1269         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1270 }
1271
1272 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1273                        enum pipe pipe, bool state)
1274 {
1275         u32 val;
1276         bool cur_state;
1277
1278         val = I915_READ(FDI_RX_CTL(pipe));
1279         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1280         I915_STATE_WARN(cur_state != state,
1281              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1282              state_string(state), state_string(cur_state));
1283 }
1284
1285 void assert_panel_unlocked(struct drm_i915_private *dev_priv,
1286                            enum pipe pipe)
1287 {
1288         struct drm_device *dev = dev_priv->dev;
1289         int pp_reg;
1290         u32 val;
1291         enum pipe panel_pipe = PIPE_A;
1292         bool locked = true;
1293
1294         if (WARN_ON(HAS_DDI(dev)))
1295                 return;
1296
1297         if (HAS_PCH_SPLIT(dev)) {
1298                 u32 port_sel;
1299
1300                 pp_reg = PCH_PP_CONTROL;
1301                 port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
1302
1303                 if (port_sel == PANEL_PORT_SELECT_LVDS &&
1304                     I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
1305                         panel_pipe = PIPE_B;
1306                 /* XXX: else fix for eDP */
1307         } else if (IS_VALLEYVIEW(dev)) {
1308                 /* presumably write lock depends on pipe, not port select */
1309                 pp_reg = VLV_PIPE_PP_CONTROL(pipe);
1310                 panel_pipe = pipe;
1311         } else {
1312                 pp_reg = PP_CONTROL;
1313                 if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
1314                         panel_pipe = PIPE_B;
1315         }
1316
1317         val = I915_READ(pp_reg);
1318         if (!(val & PANEL_POWER_ON) ||
1319             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1320                 locked = false;
1321
1322         I915_STATE_WARN(panel_pipe == pipe && locked,
1323              "panel assertion failure, pipe %c regs locked\n",
1324              pipe_name(pipe));
1325 }
1326
1327 static void assert_cursor(struct drm_i915_private *dev_priv,
1328                           enum pipe pipe, bool state)
1329 {
1330         struct drm_device *dev = dev_priv->dev;
1331         bool cur_state;
1332
1333         if (IS_845G(dev) || IS_I865G(dev))
1334                 cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
1335         else
1336                 cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
1337
1338         I915_STATE_WARN(cur_state != state,
1339              "cursor on pipe %c assertion failure (expected %s, current %s)\n",
1340              pipe_name(pipe), state_string(state), state_string(cur_state));
1341 }
1342 #define assert_cursor_enabled(d, p) assert_cursor(d, p, true)
1343 #define assert_cursor_disabled(d, p) assert_cursor(d, p, false)
1344
1345 void assert_pipe(struct drm_i915_private *dev_priv,
1346                  enum pipe pipe, bool state)
1347 {
1348         bool cur_state;
1349         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1350                                                                       pipe);
1351
1352         /* if we need the pipe quirk it must be always on */
1353         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1354             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1355                 state = true;
1356
1357         if (!intel_display_power_is_enabled(dev_priv,
1358                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
1359                 cur_state = false;
1360         } else {
1361                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1362                 cur_state = !!(val & PIPECONF_ENABLE);
1363         }
1364
1365         I915_STATE_WARN(cur_state != state,
1366              "pipe %c assertion failure (expected %s, current %s)\n",
1367              pipe_name(pipe), state_string(state), state_string(cur_state));
1368 }
1369
1370 static void assert_plane(struct drm_i915_private *dev_priv,
1371                          enum plane plane, bool state)
1372 {
1373         u32 val;
1374         bool cur_state;
1375
1376         val = I915_READ(DSPCNTR(plane));
1377         cur_state = !!(val & DISPLAY_PLANE_ENABLE);
1378         I915_STATE_WARN(cur_state != state,
1379              "plane %c assertion failure (expected %s, current %s)\n",
1380              plane_name(plane), state_string(state), state_string(cur_state));
1381 }
1382
1383 #define assert_plane_enabled(d, p) assert_plane(d, p, true)
1384 #define assert_plane_disabled(d, p) assert_plane(d, p, false)
1385
1386 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1387                                    enum pipe pipe)
1388 {
1389         struct drm_device *dev = dev_priv->dev;
1390         int i;
1391
1392         /* Primary planes are fixed to pipes on gen4+ */
1393         if (INTEL_INFO(dev)->gen >= 4) {
1394                 u32 val = I915_READ(DSPCNTR(pipe));
1395                 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
1396                      "plane %c assertion failure, should be disabled but not\n",
1397                      plane_name(pipe));
1398                 return;
1399         }
1400
1401         /* Need to check both planes against the pipe */
1402         for_each_pipe(dev_priv, i) {
1403                 u32 val = I915_READ(DSPCNTR(i));
1404                 enum pipe cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
1405                         DISPPLANE_SEL_PIPE_SHIFT;
1406                 I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
1407                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
1408                      plane_name(i), pipe_name(pipe));
1409         }
1410 }
1411
1412 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1413                                     enum pipe pipe)
1414 {
1415         struct drm_device *dev = dev_priv->dev;
1416         int sprite;
1417
1418         if (INTEL_INFO(dev)->gen >= 9) {
1419                 for_each_sprite(dev_priv, pipe, sprite) {
1420                         u32 val = I915_READ(PLANE_CTL(pipe, sprite));
1421                         I915_STATE_WARN(val & PLANE_CTL_ENABLE,
1422                              "plane %d assertion failure, should be off on pipe %c but is still active\n",
1423                              sprite, pipe_name(pipe));
1424                 }
1425         } else if (IS_VALLEYVIEW(dev)) {
1426                 for_each_sprite(dev_priv, pipe, sprite) {
1427                         u32 val = I915_READ(SPCNTR(pipe, sprite));
1428                         I915_STATE_WARN(val & SP_ENABLE,
1429                              "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1430                              sprite_name(pipe, sprite), pipe_name(pipe));
1431                 }
1432         } else if (INTEL_INFO(dev)->gen >= 7) {
1433                 u32 val = I915_READ(SPRCTL(pipe));
1434                 I915_STATE_WARN(val & SPRITE_ENABLE,
1435                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1436                      plane_name(pipe), pipe_name(pipe));
1437         } else if (INTEL_INFO(dev)->gen >= 5) {
1438                 u32 val = I915_READ(DVSCNTR(pipe));
1439                 I915_STATE_WARN(val & DVS_ENABLE,
1440                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1441                      plane_name(pipe), pipe_name(pipe));
1442         }
1443 }
1444
1445 static void assert_vblank_disabled(struct drm_crtc *crtc)
1446 {
1447         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1448                 drm_crtc_vblank_put(crtc);
1449 }
1450
1451 static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
1452 {
1453         u32 val;
1454         bool enabled;
1455
1456         I915_STATE_WARN_ON(!(HAS_PCH_IBX(dev_priv->dev) || HAS_PCH_CPT(dev_priv->dev)));
1457
1458         val = I915_READ(PCH_DREF_CONTROL);
1459         enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
1460                             DREF_SUPERSPREAD_SOURCE_MASK));
1461         I915_STATE_WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
1462 }
1463
1464 static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1465                                            enum pipe pipe)
1466 {
1467         u32 val;
1468         bool enabled;
1469
1470         val = I915_READ(PCH_TRANSCONF(pipe));
1471         enabled = !!(val & TRANS_ENABLE);
1472         I915_STATE_WARN(enabled,
1473              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1474              pipe_name(pipe));
1475 }
1476
1477 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
1478                             enum pipe pipe, u32 port_sel, u32 val)
1479 {
1480         if ((val & DP_PORT_EN) == 0)
1481                 return false;
1482
1483         if (HAS_PCH_CPT(dev_priv->dev)) {
1484                 u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
1485                 u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
1486                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
1487                         return false;
1488         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1489                 if ((val & DP_PIPE_MASK_CHV) != DP_PIPE_SELECT_CHV(pipe))
1490                         return false;
1491         } else {
1492                 if ((val & DP_PIPE_MASK) != (pipe << 30))
1493                         return false;
1494         }
1495         return true;
1496 }
1497
1498 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1499                               enum pipe pipe, u32 val)
1500 {
1501         if ((val & SDVO_ENABLE) == 0)
1502                 return false;
1503
1504         if (HAS_PCH_CPT(dev_priv->dev)) {
1505                 if ((val & SDVO_PIPE_SEL_MASK_CPT) != SDVO_PIPE_SEL_CPT(pipe))
1506                         return false;
1507         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1508                 if ((val & SDVO_PIPE_SEL_MASK_CHV) != SDVO_PIPE_SEL_CHV(pipe))
1509                         return false;
1510         } else {
1511                 if ((val & SDVO_PIPE_SEL_MASK) != SDVO_PIPE_SEL(pipe))
1512                         return false;
1513         }
1514         return true;
1515 }
1516
1517 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1518                               enum pipe pipe, u32 val)
1519 {
1520         if ((val & LVDS_PORT_EN) == 0)
1521                 return false;
1522
1523         if (HAS_PCH_CPT(dev_priv->dev)) {
1524                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1525                         return false;
1526         } else {
1527                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1528                         return false;
1529         }
1530         return true;
1531 }
1532
1533 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1534                               enum pipe pipe, u32 val)
1535 {
1536         if ((val & ADPA_DAC_ENABLE) == 0)
1537                 return false;
1538         if (HAS_PCH_CPT(dev_priv->dev)) {
1539                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1540                         return false;
1541         } else {
1542                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1543                         return false;
1544         }
1545         return true;
1546 }
1547
1548 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1549                                    enum pipe pipe, int reg, u32 port_sel)
1550 {
1551         u32 val = I915_READ(reg);
1552         I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1553              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1554              reg, pipe_name(pipe));
1555
1556         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & DP_PORT_EN) == 0
1557              && (val & DP_PIPEB_SELECT),
1558              "IBX PCH dp port still using transcoder B\n");
1559 }
1560
1561 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1562                                      enum pipe pipe, int reg)
1563 {
1564         u32 val = I915_READ(reg);
1565         I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val),
1566              "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
1567              reg, pipe_name(pipe));
1568
1569         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & SDVO_ENABLE) == 0
1570              && (val & SDVO_PIPE_B_SELECT),
1571              "IBX PCH hdmi port still using transcoder B\n");
1572 }
1573
1574 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1575                                       enum pipe pipe)
1576 {
1577         u32 val;
1578
1579         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1580         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1581         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1582
1583         val = I915_READ(PCH_ADPA);
1584         I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val),
1585              "PCH VGA enabled on transcoder %c, should be disabled\n",
1586              pipe_name(pipe));
1587
1588         val = I915_READ(PCH_LVDS);
1589         I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val),
1590              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1591              pipe_name(pipe));
1592
1593         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIB);
1594         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIC);
1595         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID);
1596 }
1597
1598 static void vlv_enable_pll(struct intel_crtc *crtc,
1599                            const struct intel_crtc_state *pipe_config)
1600 {
1601         struct drm_device *dev = crtc->base.dev;
1602         struct drm_i915_private *dev_priv = dev->dev_private;
1603         int reg = DPLL(crtc->pipe);
1604         u32 dpll = pipe_config->dpll_hw_state.dpll;
1605
1606         assert_pipe_disabled(dev_priv, crtc->pipe);
1607
1608         /* No really, not for ILK+ */
1609         BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
1610
1611         /* PLL is protected by panel, make sure we can write it */
1612         if (IS_MOBILE(dev_priv->dev))
1613                 assert_panel_unlocked(dev_priv, crtc->pipe);
1614
1615         I915_WRITE(reg, dpll);
1616         POSTING_READ(reg);
1617         udelay(150);
1618
1619         if (wait_for(((I915_READ(reg) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1620                 DRM_ERROR("DPLL %d failed to lock\n", crtc->pipe);
1621
1622         I915_WRITE(DPLL_MD(crtc->pipe), pipe_config->dpll_hw_state.dpll_md);
1623         POSTING_READ(DPLL_MD(crtc->pipe));
1624
1625         /* We do this three times for luck */
1626         I915_WRITE(reg, dpll);
1627         POSTING_READ(reg);
1628         udelay(150); /* wait for warmup */
1629         I915_WRITE(reg, dpll);
1630         POSTING_READ(reg);
1631         udelay(150); /* wait for warmup */
1632         I915_WRITE(reg, dpll);
1633         POSTING_READ(reg);
1634         udelay(150); /* wait for warmup */
1635 }
1636
1637 static void chv_enable_pll(struct intel_crtc *crtc,
1638                            const struct intel_crtc_state *pipe_config)
1639 {
1640         struct drm_device *dev = crtc->base.dev;
1641         struct drm_i915_private *dev_priv = dev->dev_private;
1642         int pipe = crtc->pipe;
1643         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1644         u32 tmp;
1645
1646         assert_pipe_disabled(dev_priv, crtc->pipe);
1647
1648         BUG_ON(!IS_CHERRYVIEW(dev_priv->dev));
1649
1650         mutex_lock(&dev_priv->sb_lock);
1651
1652         /* Enable back the 10bit clock to display controller */
1653         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1654         tmp |= DPIO_DCLKP_EN;
1655         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1656
1657         mutex_unlock(&dev_priv->sb_lock);
1658
1659         /*
1660          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1661          */
1662         udelay(1);
1663
1664         /* Enable PLL */
1665         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1666
1667         /* Check PLL is locked */
1668         if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1669                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1670
1671         /* not sure when this should be written */
1672         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1673         POSTING_READ(DPLL_MD(pipe));
1674 }
1675
1676 static int intel_num_dvo_pipes(struct drm_device *dev)
1677 {
1678         struct intel_crtc *crtc;
1679         int count = 0;
1680
1681         for_each_intel_crtc(dev, crtc)
1682                 count += crtc->base.state->active &&
1683                         intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO);
1684
1685         return count;
1686 }
1687
1688 static void i9xx_enable_pll(struct intel_crtc *crtc)
1689 {
1690         struct drm_device *dev = crtc->base.dev;
1691         struct drm_i915_private *dev_priv = dev->dev_private;
1692         int reg = DPLL(crtc->pipe);
1693         u32 dpll = crtc->config->dpll_hw_state.dpll;
1694
1695         assert_pipe_disabled(dev_priv, crtc->pipe);
1696
1697         /* No really, not for ILK+ */
1698         BUG_ON(INTEL_INFO(dev)->gen >= 5);
1699
1700         /* PLL is protected by panel, make sure we can write it */
1701         if (IS_MOBILE(dev) && !IS_I830(dev))
1702                 assert_panel_unlocked(dev_priv, crtc->pipe);
1703
1704         /* Enable DVO 2x clock on both PLLs if necessary */
1705         if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
1706                 /*
1707                  * It appears to be important that we don't enable this
1708                  * for the current pipe before otherwise configuring the
1709                  * PLL. No idea how this should be handled if multiple
1710                  * DVO outputs are enabled simultaneosly.
1711                  */
1712                 dpll |= DPLL_DVO_2X_MODE;
1713                 I915_WRITE(DPLL(!crtc->pipe),
1714                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1715         }
1716
1717         /*
1718          * Apparently we need to have VGA mode enabled prior to changing
1719          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1720          * dividers, even though the register value does change.
1721          */
1722         I915_WRITE(reg, 0);
1723
1724         I915_WRITE(reg, dpll);
1725
1726         /* Wait for the clocks to stabilize. */
1727         POSTING_READ(reg);
1728         udelay(150);
1729
1730         if (INTEL_INFO(dev)->gen >= 4) {
1731                 I915_WRITE(DPLL_MD(crtc->pipe),
1732                            crtc->config->dpll_hw_state.dpll_md);
1733         } else {
1734                 /* The pixel multiplier can only be updated once the
1735                  * DPLL is enabled and the clocks are stable.
1736                  *
1737                  * So write it again.
1738                  */
1739                 I915_WRITE(reg, dpll);
1740         }
1741
1742         /* We do this three times for luck */
1743         I915_WRITE(reg, dpll);
1744         POSTING_READ(reg);
1745         udelay(150); /* wait for warmup */
1746         I915_WRITE(reg, dpll);
1747         POSTING_READ(reg);
1748         udelay(150); /* wait for warmup */
1749         I915_WRITE(reg, dpll);
1750         POSTING_READ(reg);
1751         udelay(150); /* wait for warmup */
1752 }
1753
1754 /**
1755  * i9xx_disable_pll - disable a PLL
1756  * @dev_priv: i915 private structure
1757  * @pipe: pipe PLL to disable
1758  *
1759  * Disable the PLL for @pipe, making sure the pipe is off first.
1760  *
1761  * Note!  This is for pre-ILK only.
1762  */
1763 static void i9xx_disable_pll(struct intel_crtc *crtc)
1764 {
1765         struct drm_device *dev = crtc->base.dev;
1766         struct drm_i915_private *dev_priv = dev->dev_private;
1767         enum pipe pipe = crtc->pipe;
1768
1769         /* Disable DVO 2x clock on both PLLs if necessary */
1770         if (IS_I830(dev) &&
1771             intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) &&
1772             !intel_num_dvo_pipes(dev)) {
1773                 I915_WRITE(DPLL(PIPE_B),
1774                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1775                 I915_WRITE(DPLL(PIPE_A),
1776                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1777         }
1778
1779         /* Don't disable pipe or pipe PLLs if needed */
1780         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1781             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1782                 return;
1783
1784         /* Make sure the pipe isn't still relying on us */
1785         assert_pipe_disabled(dev_priv, pipe);
1786
1787         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1788         POSTING_READ(DPLL(pipe));
1789 }
1790
1791 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1792 {
1793         u32 val;
1794
1795         /* Make sure the pipe isn't still relying on us */
1796         assert_pipe_disabled(dev_priv, pipe);
1797
1798         /*
1799          * Leave integrated clock source and reference clock enabled for pipe B.
1800          * The latter is needed for VGA hotplug / manual detection.
1801          */
1802         val = DPLL_VGA_MODE_DIS;
1803         if (pipe == PIPE_B)
1804                 val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REF_CLK_ENABLE_VLV;
1805         I915_WRITE(DPLL(pipe), val);
1806         POSTING_READ(DPLL(pipe));
1807
1808 }
1809
1810 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1811 {
1812         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1813         u32 val;
1814
1815         /* Make sure the pipe isn't still relying on us */
1816         assert_pipe_disabled(dev_priv, pipe);
1817
1818         /* Set PLL en = 0 */
1819         val = DPLL_SSC_REF_CLK_CHV |
1820                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1821         if (pipe != PIPE_A)
1822                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1823         I915_WRITE(DPLL(pipe), val);
1824         POSTING_READ(DPLL(pipe));
1825
1826         mutex_lock(&dev_priv->sb_lock);
1827
1828         /* Disable 10bit clock to display controller */
1829         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1830         val &= ~DPIO_DCLKP_EN;
1831         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1832
1833         mutex_unlock(&dev_priv->sb_lock);
1834 }
1835
1836 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1837                          struct intel_digital_port *dport,
1838                          unsigned int expected_mask)
1839 {
1840         u32 port_mask;
1841         int dpll_reg;
1842
1843         switch (dport->port) {
1844         case PORT_B:
1845                 port_mask = DPLL_PORTB_READY_MASK;
1846                 dpll_reg = DPLL(0);
1847                 break;
1848         case PORT_C:
1849                 port_mask = DPLL_PORTC_READY_MASK;
1850                 dpll_reg = DPLL(0);
1851                 expected_mask <<= 4;
1852                 break;
1853         case PORT_D:
1854                 port_mask = DPLL_PORTD_READY_MASK;
1855                 dpll_reg = DPIO_PHY_STATUS;
1856                 break;
1857         default:
1858                 BUG();
1859         }
1860
1861         if (wait_for((I915_READ(dpll_reg) & port_mask) == expected_mask, 1000))
1862                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1863                      port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
1864 }
1865
1866 static void intel_prepare_shared_dpll(struct intel_crtc *crtc)
1867 {
1868         struct drm_device *dev = crtc->base.dev;
1869         struct drm_i915_private *dev_priv = dev->dev_private;
1870         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1871
1872         if (WARN_ON(pll == NULL))
1873                 return;
1874
1875         WARN_ON(!pll->config.crtc_mask);
1876         if (pll->active == 0) {
1877                 DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
1878                 WARN_ON(pll->on);
1879                 assert_shared_dpll_disabled(dev_priv, pll);
1880
1881                 pll->mode_set(dev_priv, pll);
1882         }
1883 }
1884
1885 /**
1886  * intel_enable_shared_dpll - enable PCH PLL
1887  * @dev_priv: i915 private structure
1888  * @pipe: pipe PLL to enable
1889  *
1890  * The PCH PLL needs to be enabled before the PCH transcoder, since it
1891  * drives the transcoder clock.
1892  */
1893 static void intel_enable_shared_dpll(struct intel_crtc *crtc)
1894 {
1895         struct drm_device *dev = crtc->base.dev;
1896         struct drm_i915_private *dev_priv = dev->dev_private;
1897         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1898
1899         if (WARN_ON(pll == NULL))
1900                 return;
1901
1902         if (WARN_ON(pll->config.crtc_mask == 0))
1903                 return;
1904
1905         DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n",
1906                       pll->name, pll->active, pll->on,
1907                       crtc->base.base.id);
1908
1909         if (pll->active++) {
1910                 WARN_ON(!pll->on);
1911                 assert_shared_dpll_enabled(dev_priv, pll);
1912                 return;
1913         }
1914         WARN_ON(pll->on);
1915
1916         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
1917
1918         DRM_DEBUG_KMS("enabling %s\n", pll->name);
1919         pll->enable(dev_priv, pll);
1920         pll->on = true;
1921 }
1922
1923 static void intel_disable_shared_dpll(struct intel_crtc *crtc)
1924 {
1925         struct drm_device *dev = crtc->base.dev;
1926         struct drm_i915_private *dev_priv = dev->dev_private;
1927         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1928
1929         /* PCH only available on ILK+ */
1930         if (INTEL_INFO(dev)->gen < 5)
1931                 return;
1932
1933         if (pll == NULL)
1934                 return;
1935
1936         if (WARN_ON(!(pll->config.crtc_mask & (1 << drm_crtc_index(&crtc->base)))))
1937                 return;
1938
1939         DRM_DEBUG_KMS("disable %s (active %d, on? %d) for crtc %d\n",
1940                       pll->name, pll->active, pll->on,
1941                       crtc->base.base.id);
1942
1943         if (WARN_ON(pll->active == 0)) {
1944                 assert_shared_dpll_disabled(dev_priv, pll);
1945                 return;
1946         }
1947
1948         assert_shared_dpll_enabled(dev_priv, pll);
1949         WARN_ON(!pll->on);
1950         if (--pll->active)
1951                 return;
1952
1953         DRM_DEBUG_KMS("disabling %s\n", pll->name);
1954         pll->disable(dev_priv, pll);
1955         pll->on = false;
1956
1957         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
1958 }
1959
1960 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1961                                            enum pipe pipe)
1962 {
1963         struct drm_device *dev = dev_priv->dev;
1964         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1965         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1966         uint32_t reg, val, pipeconf_val;
1967
1968         /* PCH only available on ILK+ */
1969         BUG_ON(!HAS_PCH_SPLIT(dev));
1970
1971         /* Make sure PCH DPLL is enabled */
1972         assert_shared_dpll_enabled(dev_priv,
1973                                    intel_crtc_to_shared_dpll(intel_crtc));
1974
1975         /* FDI must be feeding us bits for PCH ports */
1976         assert_fdi_tx_enabled(dev_priv, pipe);
1977         assert_fdi_rx_enabled(dev_priv, pipe);
1978
1979         if (HAS_PCH_CPT(dev)) {
1980                 /* Workaround: Set the timing override bit before enabling the
1981                  * pch transcoder. */
1982                 reg = TRANS_CHICKEN2(pipe);
1983                 val = I915_READ(reg);
1984                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1985                 I915_WRITE(reg, val);
1986         }
1987
1988         reg = PCH_TRANSCONF(pipe);
1989         val = I915_READ(reg);
1990         pipeconf_val = I915_READ(PIPECONF(pipe));
1991
1992         if (HAS_PCH_IBX(dev_priv->dev)) {
1993                 /*
1994                  * Make the BPC in transcoder be consistent with
1995                  * that in pipeconf reg. For HDMI we must use 8bpc
1996                  * here for both 8bpc and 12bpc.
1997                  */
1998                 val &= ~PIPECONF_BPC_MASK;
1999                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI))
2000                         val |= PIPECONF_8BPC;
2001                 else
2002                         val |= pipeconf_val & PIPECONF_BPC_MASK;
2003         }
2004
2005         val &= ~TRANS_INTERLACE_MASK;
2006         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
2007                 if (HAS_PCH_IBX(dev_priv->dev) &&
2008                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
2009                         val |= TRANS_LEGACY_INTERLACED_ILK;
2010                 else
2011                         val |= TRANS_INTERLACED;
2012         else
2013                 val |= TRANS_PROGRESSIVE;
2014
2015         I915_WRITE(reg, val | TRANS_ENABLE);
2016         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
2017                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
2018 }
2019
2020 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
2021                                       enum transcoder cpu_transcoder)
2022 {
2023         u32 val, pipeconf_val;
2024
2025         /* PCH only available on ILK+ */
2026         BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev));
2027
2028         /* FDI must be feeding us bits for PCH ports */
2029         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
2030         assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
2031
2032         /* Workaround: set timing override bit. */
2033         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
2034         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
2035         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
2036
2037         val = TRANS_ENABLE;
2038         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
2039
2040         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
2041             PIPECONF_INTERLACED_ILK)
2042                 val |= TRANS_INTERLACED;
2043         else
2044                 val |= TRANS_PROGRESSIVE;
2045
2046         I915_WRITE(LPT_TRANSCONF, val);
2047         if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
2048                 DRM_ERROR("Failed to enable PCH transcoder\n");
2049 }
2050
2051 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
2052                                             enum pipe pipe)
2053 {
2054         struct drm_device *dev = dev_priv->dev;
2055         uint32_t reg, val;
2056
2057         /* FDI relies on the transcoder */
2058         assert_fdi_tx_disabled(dev_priv, pipe);
2059         assert_fdi_rx_disabled(dev_priv, pipe);
2060
2061         /* Ports must be off as well */
2062         assert_pch_ports_disabled(dev_priv, pipe);
2063
2064         reg = PCH_TRANSCONF(pipe);
2065         val = I915_READ(reg);
2066         val &= ~TRANS_ENABLE;
2067         I915_WRITE(reg, val);
2068         /* wait for PCH transcoder off, transcoder state */
2069         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
2070                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
2071
2072         if (!HAS_PCH_IBX(dev)) {
2073                 /* Workaround: Clear the timing override chicken bit again. */
2074                 reg = TRANS_CHICKEN2(pipe);
2075                 val = I915_READ(reg);
2076                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2077                 I915_WRITE(reg, val);
2078         }
2079 }
2080
2081 static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
2082 {
2083         u32 val;
2084
2085         val = I915_READ(LPT_TRANSCONF);
2086         val &= ~TRANS_ENABLE;
2087         I915_WRITE(LPT_TRANSCONF, val);
2088         /* wait for PCH transcoder off, transcoder state */
2089         if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
2090                 DRM_ERROR("Failed to disable PCH transcoder\n");
2091
2092         /* Workaround: clear timing override bit. */
2093         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
2094         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2095         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
2096 }
2097
2098 /**
2099  * intel_enable_pipe - enable a pipe, asserting requirements
2100  * @crtc: crtc responsible for the pipe
2101  *
2102  * Enable @crtc's pipe, making sure that various hardware specific requirements
2103  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
2104  */
2105 static void intel_enable_pipe(struct intel_crtc *crtc)
2106 {
2107         struct drm_device *dev = crtc->base.dev;
2108         struct drm_i915_private *dev_priv = dev->dev_private;
2109         enum pipe pipe = crtc->pipe;
2110         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
2111                                                                       pipe);
2112         enum pipe pch_transcoder;
2113         int reg;
2114         u32 val;
2115
2116         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
2117
2118         assert_planes_disabled(dev_priv, pipe);
2119         assert_cursor_disabled(dev_priv, pipe);
2120         assert_sprites_disabled(dev_priv, pipe);
2121
2122         if (HAS_PCH_LPT(dev_priv->dev))
2123                 pch_transcoder = TRANSCODER_A;
2124         else
2125                 pch_transcoder = pipe;
2126
2127         /*
2128          * A pipe without a PLL won't actually be able to drive bits from
2129          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
2130          * need the check.
2131          */
2132         if (HAS_GMCH_DISPLAY(dev_priv->dev))
2133                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI))
2134                         assert_dsi_pll_enabled(dev_priv);
2135                 else
2136                         assert_pll_enabled(dev_priv, pipe);
2137         else {
2138                 if (crtc->config->has_pch_encoder) {
2139                         /* if driving the PCH, we need FDI enabled */
2140                         assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
2141                         assert_fdi_tx_pll_enabled(dev_priv,
2142                                                   (enum pipe) cpu_transcoder);
2143                 }
2144                 /* FIXME: assert CPU port conditions for SNB+ */
2145         }
2146
2147         reg = PIPECONF(cpu_transcoder);
2148         val = I915_READ(reg);
2149         if (val & PIPECONF_ENABLE) {
2150                 WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
2151                           (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
2152                 return;
2153         }
2154
2155         I915_WRITE(reg, val | PIPECONF_ENABLE);
2156         POSTING_READ(reg);
2157 }
2158
2159 /**
2160  * intel_disable_pipe - disable a pipe, asserting requirements
2161  * @crtc: crtc whose pipes is to be disabled
2162  *
2163  * Disable the pipe of @crtc, making sure that various hardware
2164  * specific requirements are met, if applicable, e.g. plane
2165  * disabled, panel fitter off, etc.
2166  *
2167  * Will wait until the pipe has shut down before returning.
2168  */
2169 static void intel_disable_pipe(struct intel_crtc *crtc)
2170 {
2171         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
2172         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
2173         enum pipe pipe = crtc->pipe;
2174         int reg;
2175         u32 val;
2176
2177         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
2178
2179         /*
2180          * Make sure planes won't keep trying to pump pixels to us,
2181          * or we might hang the display.
2182          */
2183         assert_planes_disabled(dev_priv, pipe);
2184         assert_cursor_disabled(dev_priv, pipe);
2185         assert_sprites_disabled(dev_priv, pipe);
2186
2187         reg = PIPECONF(cpu_transcoder);
2188         val = I915_READ(reg);
2189         if ((val & PIPECONF_ENABLE) == 0)
2190                 return;
2191
2192         /*
2193          * Double wide has implications for planes
2194          * so best keep it disabled when not needed.
2195          */
2196         if (crtc->config->double_wide)
2197                 val &= ~PIPECONF_DOUBLE_WIDE;
2198
2199         /* Don't disable pipe or pipe PLLs if needed */
2200         if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
2201             !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
2202                 val &= ~PIPECONF_ENABLE;
2203
2204         I915_WRITE(reg, val);
2205         if ((val & PIPECONF_ENABLE) == 0)
2206                 intel_wait_for_pipe_off(crtc);
2207 }
2208
2209 static bool need_vtd_wa(struct drm_device *dev)
2210 {
2211 #ifdef CONFIG_INTEL_IOMMU
2212         if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped)
2213                 return true;
2214 #endif
2215         return false;
2216 }
2217
2218 unsigned int
2219 intel_tile_height(struct drm_device *dev, uint32_t pixel_format,
2220                   uint64_t fb_format_modifier, unsigned int plane)
2221 {
2222         unsigned int tile_height;
2223         uint32_t pixel_bytes;
2224
2225         switch (fb_format_modifier) {
2226         case DRM_FORMAT_MOD_NONE:
2227                 tile_height = 1;
2228                 break;
2229         case I915_FORMAT_MOD_X_TILED:
2230                 tile_height = IS_GEN2(dev) ? 16 : 8;
2231                 break;
2232         case I915_FORMAT_MOD_Y_TILED:
2233                 tile_height = 32;
2234                 break;
2235         case I915_FORMAT_MOD_Yf_TILED:
2236                 pixel_bytes = drm_format_plane_cpp(pixel_format, plane);
2237                 switch (pixel_bytes) {
2238                 default:
2239                 case 1:
2240                         tile_height = 64;
2241                         break;
2242                 case 2:
2243                 case 4:
2244                         tile_height = 32;
2245                         break;
2246                 case 8:
2247                         tile_height = 16;
2248                         break;
2249                 case 16:
2250                         WARN_ONCE(1,
2251                                   "128-bit pixels are not supported for display!");
2252                         tile_height = 16;
2253                         break;
2254                 }
2255                 break;
2256         default:
2257                 MISSING_CASE(fb_format_modifier);
2258                 tile_height = 1;
2259                 break;
2260         }
2261
2262         return tile_height;
2263 }
2264
2265 unsigned int
2266 intel_fb_align_height(struct drm_device *dev, unsigned int height,
2267                       uint32_t pixel_format, uint64_t fb_format_modifier)
2268 {
2269         return ALIGN(height, intel_tile_height(dev, pixel_format,
2270                                                fb_format_modifier, 0));
2271 }
2272
2273 static int
2274 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb,
2275                         const struct drm_plane_state *plane_state)
2276 {
2277         struct intel_rotation_info *info = &view->rotation_info;
2278         unsigned int tile_height, tile_pitch;
2279
2280         *view = i915_ggtt_view_normal;
2281
2282         if (!plane_state)
2283                 return 0;
2284
2285         if (!intel_rotation_90_or_270(plane_state->rotation))
2286                 return 0;
2287
2288         *view = i915_ggtt_view_rotated;
2289
2290         info->height = fb->height;
2291         info->pixel_format = fb->pixel_format;
2292         info->pitch = fb->pitches[0];
2293         info->uv_offset = fb->offsets[1];
2294         info->fb_modifier = fb->modifier[0];
2295
2296         tile_height = intel_tile_height(fb->dev, fb->pixel_format,
2297                                         fb->modifier[0], 0);
2298         tile_pitch = PAGE_SIZE / tile_height;
2299         info->width_pages = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
2300         info->height_pages = DIV_ROUND_UP(fb->height, tile_height);
2301         info->size = info->width_pages * info->height_pages * PAGE_SIZE;
2302
2303         if (info->pixel_format == DRM_FORMAT_NV12) {
2304                 tile_height = intel_tile_height(fb->dev, fb->pixel_format,
2305                                                 fb->modifier[0], 1);
2306                 tile_pitch = PAGE_SIZE / tile_height;
2307                 info->width_pages_uv = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
2308                 info->height_pages_uv = DIV_ROUND_UP(fb->height / 2,
2309                                                      tile_height);
2310                 info->size_uv = info->width_pages_uv * info->height_pages_uv *
2311                                 PAGE_SIZE;
2312         }
2313
2314         return 0;
2315 }
2316
2317 static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
2318 {
2319         if (INTEL_INFO(dev_priv)->gen >= 9)
2320                 return 256 * 1024;
2321         else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
2322                  IS_VALLEYVIEW(dev_priv))
2323                 return 128 * 1024;
2324         else if (INTEL_INFO(dev_priv)->gen >= 4)
2325                 return 4 * 1024;
2326         else
2327                 return 0;
2328 }
2329
2330 int
2331 intel_pin_and_fence_fb_obj(struct drm_plane *plane,
2332                            struct drm_framebuffer *fb,
2333                            const struct drm_plane_state *plane_state,
2334                            struct intel_engine_cs *pipelined,
2335                            struct drm_i915_gem_request **pipelined_request)
2336 {
2337         struct drm_device *dev = fb->dev;
2338         struct drm_i915_private *dev_priv = dev->dev_private;
2339         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2340         struct i915_ggtt_view view;
2341         u32 alignment;
2342         int ret;
2343
2344         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2345
2346         switch (fb->modifier[0]) {
2347         case DRM_FORMAT_MOD_NONE:
2348                 alignment = intel_linear_alignment(dev_priv);
2349                 break;
2350         case I915_FORMAT_MOD_X_TILED:
2351                 if (INTEL_INFO(dev)->gen >= 9)
2352                         alignment = 256 * 1024;
2353                 else {
2354                         /* pin() will align the object as required by fence */
2355                         alignment = 0;
2356                 }
2357                 break;
2358         case I915_FORMAT_MOD_Y_TILED:
2359         case I915_FORMAT_MOD_Yf_TILED:
2360                 if (WARN_ONCE(INTEL_INFO(dev)->gen < 9,
2361                           "Y tiling bo slipped through, driver bug!\n"))
2362                         return -EINVAL;
2363                 alignment = 1 * 1024 * 1024;
2364                 break;
2365         default:
2366                 MISSING_CASE(fb->modifier[0]);
2367                 return -EINVAL;
2368         }
2369
2370         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2371         if (ret)
2372                 return ret;
2373
2374         /* Note that the w/a also requires 64 PTE of padding following the
2375          * bo. We currently fill all unused PTE with the shadow page and so
2376          * we should always have valid PTE following the scanout preventing
2377          * the VT-d warning.
2378          */
2379         if (need_vtd_wa(dev) && alignment < 256 * 1024)
2380                 alignment = 256 * 1024;
2381
2382         /*
2383          * Global gtt pte registers are special registers which actually forward
2384          * writes to a chunk of system memory. Which means that there is no risk
2385          * that the register values disappear as soon as we call
2386          * intel_runtime_pm_put(), so it is correct to wrap only the
2387          * pin/unpin/fence and not more.
2388          */
2389         intel_runtime_pm_get(dev_priv);
2390
2391         dev_priv->mm.interruptible = false;
2392         ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined,
2393                                                    pipelined_request, &view);
2394         if (ret)
2395                 goto err_interruptible;
2396
2397         /* Install a fence for tiled scan-out. Pre-i965 always needs a
2398          * fence, whereas 965+ only requires a fence if using
2399          * framebuffer compression.  For simplicity, we always install
2400          * a fence as the cost is not that onerous.
2401          */
2402         if (view.type == I915_GGTT_VIEW_NORMAL) {
2403                 ret = i915_gem_object_get_fence(obj);
2404                 if (ret == -EDEADLK) {
2405                         /*
2406                          * -EDEADLK means there are no free fences
2407                          * no pending flips.
2408                          *
2409                          * This is propagated to atomic, but it uses
2410                          * -EDEADLK to force a locking recovery, so
2411                          * change the returned error to -EBUSY.
2412                          */
2413                         ret = -EBUSY;
2414                         goto err_unpin;
2415                 } else if (ret)
2416                         goto err_unpin;
2417
2418                 i915_gem_object_pin_fence(obj);
2419         }
2420
2421         dev_priv->mm.interruptible = true;
2422         intel_runtime_pm_put(dev_priv);
2423         return 0;
2424
2425 err_unpin:
2426         i915_gem_object_unpin_from_display_plane(obj, &view);
2427 err_interruptible:
2428         dev_priv->mm.interruptible = true;
2429         intel_runtime_pm_put(dev_priv);
2430         return ret;
2431 }
2432
2433 static void intel_unpin_fb_obj(struct drm_framebuffer *fb,
2434                                const struct drm_plane_state *plane_state)
2435 {
2436         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2437         struct i915_ggtt_view view;
2438         int ret;
2439
2440         WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
2441
2442         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2443         WARN_ONCE(ret, "Couldn't get view from plane state!");
2444
2445         if (view.type == I915_GGTT_VIEW_NORMAL)
2446                 i915_gem_object_unpin_fence(obj);
2447
2448         i915_gem_object_unpin_from_display_plane(obj, &view);
2449 }
2450
2451 /* Computes the linear offset to the base tile and adjusts x, y. bytes per pixel
2452  * is assumed to be a power-of-two. */
2453 unsigned long intel_gen4_compute_page_offset(struct drm_i915_private *dev_priv,
2454                                              int *x, int *y,
2455                                              unsigned int tiling_mode,
2456                                              unsigned int cpp,
2457                                              unsigned int pitch)
2458 {
2459         if (tiling_mode != I915_TILING_NONE) {
2460                 unsigned int tile_rows, tiles;
2461
2462                 tile_rows = *y / 8;
2463                 *y %= 8;
2464
2465                 tiles = *x / (512/cpp);
2466                 *x %= 512/cpp;
2467
2468                 return tile_rows * pitch * 8 + tiles * 4096;
2469         } else {
2470                 unsigned int alignment = intel_linear_alignment(dev_priv) - 1;
2471                 unsigned int offset;
2472
2473                 offset = *y * pitch + *x * cpp;
2474                 *y = (offset & alignment) / pitch;
2475                 *x = ((offset & alignment) - *y * pitch) / cpp;
2476                 return offset & ~alignment;
2477         }
2478 }
2479
2480 static int i9xx_format_to_fourcc(int format)
2481 {
2482         switch (format) {
2483         case DISPPLANE_8BPP:
2484                 return DRM_FORMAT_C8;
2485         case DISPPLANE_BGRX555:
2486                 return DRM_FORMAT_XRGB1555;
2487         case DISPPLANE_BGRX565:
2488                 return DRM_FORMAT_RGB565;
2489         default:
2490         case DISPPLANE_BGRX888:
2491                 return DRM_FORMAT_XRGB8888;
2492         case DISPPLANE_RGBX888:
2493                 return DRM_FORMAT_XBGR8888;
2494         case DISPPLANE_BGRX101010:
2495                 return DRM_FORMAT_XRGB2101010;
2496         case DISPPLANE_RGBX101010:
2497                 return DRM_FORMAT_XBGR2101010;
2498         }
2499 }
2500
2501 static int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2502 {
2503         switch (format) {
2504         case PLANE_CTL_FORMAT_RGB_565:
2505                 return DRM_FORMAT_RGB565;
2506         default:
2507         case PLANE_CTL_FORMAT_XRGB_8888:
2508                 if (rgb_order) {
2509                         if (alpha)
2510                                 return DRM_FORMAT_ABGR8888;
2511                         else
2512                                 return DRM_FORMAT_XBGR8888;
2513                 } else {
2514                         if (alpha)
2515                                 return DRM_FORMAT_ARGB8888;
2516                         else
2517                                 return DRM_FORMAT_XRGB8888;
2518                 }
2519         case PLANE_CTL_FORMAT_XRGB_2101010:
2520                 if (rgb_order)
2521                         return DRM_FORMAT_XBGR2101010;
2522                 else
2523                         return DRM_FORMAT_XRGB2101010;
2524         }
2525 }
2526
2527 static bool
2528 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2529                               struct intel_initial_plane_config *plane_config)
2530 {
2531         struct drm_device *dev = crtc->base.dev;
2532         struct drm_i915_private *dev_priv = to_i915(dev);
2533         struct drm_i915_gem_object *obj = NULL;
2534         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2535         struct drm_framebuffer *fb = &plane_config->fb->base;
2536         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2537         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2538                                     PAGE_SIZE);
2539
2540         size_aligned -= base_aligned;
2541
2542         if (plane_config->size == 0)
2543                 return false;
2544
2545         /* If the FB is too big, just don't use it since fbdev is not very
2546          * important and we should probably use that space with FBC or other
2547          * features. */
2548         if (size_aligned * 2 > dev_priv->gtt.stolen_usable_size)
2549                 return false;
2550
2551         obj = i915_gem_object_create_stolen_for_preallocated(dev,
2552                                                              base_aligned,
2553                                                              base_aligned,
2554                                                              size_aligned);
2555         if (!obj)
2556                 return false;
2557
2558         obj->tiling_mode = plane_config->tiling;
2559         if (obj->tiling_mode == I915_TILING_X)
2560                 obj->stride = fb->pitches[0];
2561
2562         mode_cmd.pixel_format = fb->pixel_format;
2563         mode_cmd.width = fb->width;
2564         mode_cmd.height = fb->height;
2565         mode_cmd.pitches[0] = fb->pitches[0];
2566         mode_cmd.modifier[0] = fb->modifier[0];
2567         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2568
2569         mutex_lock(&dev->struct_mutex);
2570         if (intel_framebuffer_init(dev, to_intel_framebuffer(fb),
2571                                    &mode_cmd, obj)) {
2572                 DRM_DEBUG_KMS("intel fb init failed\n");
2573                 goto out_unref_obj;
2574         }
2575         mutex_unlock(&dev->struct_mutex);
2576
2577         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2578         return true;
2579
2580 out_unref_obj:
2581         drm_gem_object_unreference(&obj->base);
2582         mutex_unlock(&dev->struct_mutex);
2583         return false;
2584 }
2585
2586 /* Update plane->state->fb to match plane->fb after driver-internal updates */
2587 static void
2588 update_state_fb(struct drm_plane *plane)
2589 {
2590         if (plane->fb == plane->state->fb)
2591                 return;
2592
2593         if (plane->state->fb)
2594                 drm_framebuffer_unreference(plane->state->fb);
2595         plane->state->fb = plane->fb;
2596         if (plane->state->fb)
2597                 drm_framebuffer_reference(plane->state->fb);
2598 }
2599
2600 static void
2601 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2602                              struct intel_initial_plane_config *plane_config)
2603 {
2604         struct drm_device *dev = intel_crtc->base.dev;
2605         struct drm_i915_private *dev_priv = dev->dev_private;
2606         struct drm_crtc *c;
2607         struct intel_crtc *i;
2608         struct drm_i915_gem_object *obj;
2609         struct drm_plane *primary = intel_crtc->base.primary;
2610         struct drm_plane_state *plane_state = primary->state;
2611         struct drm_crtc_state *crtc_state = intel_crtc->base.state;
2612         struct intel_plane *intel_plane = to_intel_plane(primary);
2613         struct drm_framebuffer *fb;
2614
2615         if (!plane_config->fb)
2616                 return;
2617
2618         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2619                 fb = &plane_config->fb->base;
2620                 goto valid_fb;
2621         }
2622
2623         kfree(plane_config->fb);
2624
2625         /*
2626          * Failed to alloc the obj, check to see if we should share
2627          * an fb with another CRTC instead
2628          */
2629         for_each_crtc(dev, c) {
2630                 i = to_intel_crtc(c);
2631
2632                 if (c == &intel_crtc->base)
2633                         continue;
2634
2635                 if (!i->active)
2636                         continue;
2637
2638                 fb = c->primary->fb;
2639                 if (!fb)
2640                         continue;
2641
2642                 obj = intel_fb_obj(fb);
2643                 if (i915_gem_obj_ggtt_offset(obj) == plane_config->base) {
2644                         drm_framebuffer_reference(fb);
2645                         goto valid_fb;
2646                 }
2647         }
2648
2649         /*
2650          * We've failed to reconstruct the BIOS FB.  Current display state
2651          * indicates that the primary plane is visible, but has a NULL FB,
2652          * which will lead to problems later if we don't fix it up.  The
2653          * simplest solution is to just disable the primary plane now and
2654          * pretend the BIOS never had it enabled.
2655          */
2656         to_intel_plane_state(plane_state)->visible = false;
2657         crtc_state->plane_mask &= ~(1 << drm_plane_index(primary));
2658         intel_pre_disable_primary(&intel_crtc->base);
2659         intel_plane->disable_plane(primary, &intel_crtc->base);
2660
2661         return;
2662
2663 valid_fb:
2664         plane_state->src_x = 0;
2665         plane_state->src_y = 0;
2666         plane_state->src_w = fb->width << 16;
2667         plane_state->src_h = fb->height << 16;
2668
2669         plane_state->crtc_x = 0;
2670         plane_state->crtc_y = 0;
2671         plane_state->crtc_w = fb->width;
2672         plane_state->crtc_h = fb->height;
2673
2674         obj = intel_fb_obj(fb);
2675         if (obj->tiling_mode != I915_TILING_NONE)
2676                 dev_priv->preserve_bios_swizzle = true;
2677
2678         drm_framebuffer_reference(fb);
2679         primary->fb = primary->state->fb = fb;
2680         primary->crtc = primary->state->crtc = &intel_crtc->base;
2681         intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary));
2682         obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit;
2683 }
2684
2685 static void i9xx_update_primary_plane(struct drm_crtc *crtc,
2686                                       struct drm_framebuffer *fb,
2687                                       int x, int y)
2688 {
2689         struct drm_device *dev = crtc->dev;
2690         struct drm_i915_private *dev_priv = dev->dev_private;
2691         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2692         struct drm_plane *primary = crtc->primary;
2693         bool visible = to_intel_plane_state(primary->state)->visible;
2694         struct drm_i915_gem_object *obj;
2695         int plane = intel_crtc->plane;
2696         unsigned long linear_offset;
2697         u32 dspcntr;
2698         u32 reg = DSPCNTR(plane);
2699         int pixel_size;
2700
2701         if (!visible || !fb) {
2702                 I915_WRITE(reg, 0);
2703                 if (INTEL_INFO(dev)->gen >= 4)
2704                         I915_WRITE(DSPSURF(plane), 0);
2705                 else
2706                         I915_WRITE(DSPADDR(plane), 0);
2707                 POSTING_READ(reg);
2708                 return;
2709         }
2710
2711         obj = intel_fb_obj(fb);
2712         if (WARN_ON(obj == NULL))
2713                 return;
2714
2715         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2716
2717         dspcntr = DISPPLANE_GAMMA_ENABLE;
2718
2719         dspcntr |= DISPLAY_PLANE_ENABLE;
2720
2721         if (INTEL_INFO(dev)->gen < 4) {
2722                 if (intel_crtc->pipe == PIPE_B)
2723                         dspcntr |= DISPPLANE_SEL_PIPE_B;
2724
2725                 /* pipesrc and dspsize control the size that is scaled from,
2726                  * which should always be the user's requested size.
2727                  */
2728                 I915_WRITE(DSPSIZE(plane),
2729                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2730                            (intel_crtc->config->pipe_src_w - 1));
2731                 I915_WRITE(DSPPOS(plane), 0);
2732         } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
2733                 I915_WRITE(PRIMSIZE(plane),
2734                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2735                            (intel_crtc->config->pipe_src_w - 1));
2736                 I915_WRITE(PRIMPOS(plane), 0);
2737                 I915_WRITE(PRIMCNSTALPHA(plane), 0);
2738         }
2739
2740         switch (fb->pixel_format) {
2741         case DRM_FORMAT_C8:
2742                 dspcntr |= DISPPLANE_8BPP;
2743                 break;
2744         case DRM_FORMAT_XRGB1555:
2745                 dspcntr |= DISPPLANE_BGRX555;
2746                 break;
2747         case DRM_FORMAT_RGB565:
2748                 dspcntr |= DISPPLANE_BGRX565;
2749                 break;
2750         case DRM_FORMAT_XRGB8888:
2751                 dspcntr |= DISPPLANE_BGRX888;
2752                 break;
2753         case DRM_FORMAT_XBGR8888:
2754                 dspcntr |= DISPPLANE_RGBX888;
2755                 break;
2756         case DRM_FORMAT_XRGB2101010:
2757                 dspcntr |= DISPPLANE_BGRX101010;
2758                 break;
2759         case DRM_FORMAT_XBGR2101010:
2760                 dspcntr |= DISPPLANE_RGBX101010;
2761                 break;
2762         default:
2763                 BUG();
2764         }
2765
2766         if (INTEL_INFO(dev)->gen >= 4 &&
2767             obj->tiling_mode != I915_TILING_NONE)
2768                 dspcntr |= DISPPLANE_TILED;
2769
2770         if (IS_G4X(dev))
2771                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2772
2773         linear_offset = y * fb->pitches[0] + x * pixel_size;
2774
2775         if (INTEL_INFO(dev)->gen >= 4) {
2776                 intel_crtc->dspaddr_offset =
2777                         intel_gen4_compute_page_offset(dev_priv,
2778                                                        &x, &y, obj->tiling_mode,
2779                                                        pixel_size,
2780                                                        fb->pitches[0]);
2781                 linear_offset -= intel_crtc->dspaddr_offset;
2782         } else {
2783                 intel_crtc->dspaddr_offset = linear_offset;
2784         }
2785
2786         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2787                 dspcntr |= DISPPLANE_ROTATE_180;
2788
2789                 x += (intel_crtc->config->pipe_src_w - 1);
2790                 y += (intel_crtc->config->pipe_src_h - 1);
2791
2792                 /* Finding the last pixel of the last line of the display
2793                 data and adding to linear_offset*/
2794                 linear_offset +=
2795                         (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2796                         (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2797         }
2798
2799         intel_crtc->adjusted_x = x;
2800         intel_crtc->adjusted_y = y;
2801
2802         I915_WRITE(reg, dspcntr);
2803
2804         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2805         if (INTEL_INFO(dev)->gen >= 4) {
2806                 I915_WRITE(DSPSURF(plane),
2807                            i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2808                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2809                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2810         } else
2811                 I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset);
2812         POSTING_READ(reg);
2813 }
2814
2815 static void ironlake_update_primary_plane(struct drm_crtc *crtc,
2816                                           struct drm_framebuffer *fb,
2817                                           int x, int y)
2818 {
2819         struct drm_device *dev = crtc->dev;
2820         struct drm_i915_private *dev_priv = dev->dev_private;
2821         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2822         struct drm_plane *primary = crtc->primary;
2823         bool visible = to_intel_plane_state(primary->state)->visible;
2824         struct drm_i915_gem_object *obj;
2825         int plane = intel_crtc->plane;
2826         unsigned long linear_offset;
2827         u32 dspcntr;
2828         u32 reg = DSPCNTR(plane);
2829         int pixel_size;
2830
2831         if (!visible || !fb) {
2832                 I915_WRITE(reg, 0);
2833                 I915_WRITE(DSPSURF(plane), 0);
2834                 POSTING_READ(reg);
2835                 return;
2836         }
2837
2838         obj = intel_fb_obj(fb);
2839         if (WARN_ON(obj == NULL))
2840                 return;
2841
2842         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2843
2844         dspcntr = DISPPLANE_GAMMA_ENABLE;
2845
2846         dspcntr |= DISPLAY_PLANE_ENABLE;
2847
2848         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2849                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
2850
2851         switch (fb->pixel_format) {
2852         case DRM_FORMAT_C8:
2853                 dspcntr |= DISPPLANE_8BPP;
2854                 break;
2855         case DRM_FORMAT_RGB565:
2856                 dspcntr |= DISPPLANE_BGRX565;
2857                 break;
2858         case DRM_FORMAT_XRGB8888:
2859                 dspcntr |= DISPPLANE_BGRX888;
2860                 break;
2861         case DRM_FORMAT_XBGR8888:
2862                 dspcntr |= DISPPLANE_RGBX888;
2863                 break;
2864         case DRM_FORMAT_XRGB2101010:
2865                 dspcntr |= DISPPLANE_BGRX101010;
2866                 break;
2867         case DRM_FORMAT_XBGR2101010:
2868                 dspcntr |= DISPPLANE_RGBX101010;
2869                 break;
2870         default:
2871                 BUG();
2872         }
2873
2874         if (obj->tiling_mode != I915_TILING_NONE)
2875                 dspcntr |= DISPPLANE_TILED;
2876
2877         if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
2878                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2879
2880         linear_offset = y * fb->pitches[0] + x * pixel_size;
2881         intel_crtc->dspaddr_offset =
2882                 intel_gen4_compute_page_offset(dev_priv,
2883                                                &x, &y, obj->tiling_mode,
2884                                                pixel_size,
2885                                                fb->pitches[0]);
2886         linear_offset -= intel_crtc->dspaddr_offset;
2887         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2888                 dspcntr |= DISPPLANE_ROTATE_180;
2889
2890                 if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
2891                         x += (intel_crtc->config->pipe_src_w - 1);
2892                         y += (intel_crtc->config->pipe_src_h - 1);
2893
2894                         /* Finding the last pixel of the last line of the display
2895                         data and adding to linear_offset*/
2896                         linear_offset +=
2897                                 (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2898                                 (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2899                 }
2900         }
2901
2902         intel_crtc->adjusted_x = x;
2903         intel_crtc->adjusted_y = y;
2904
2905         I915_WRITE(reg, dspcntr);
2906
2907         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2908         I915_WRITE(DSPSURF(plane),
2909                    i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2910         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2911                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
2912         } else {
2913                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2914                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2915         }
2916         POSTING_READ(reg);
2917 }
2918
2919 u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier,
2920                               uint32_t pixel_format)
2921 {
2922         u32 bits_per_pixel = drm_format_plane_cpp(pixel_format, 0) * 8;
2923
2924         /*
2925          * The stride is either expressed as a multiple of 64 bytes
2926          * chunks for linear buffers or in number of tiles for tiled
2927          * buffers.
2928          */
2929         switch (fb_modifier) {
2930         case DRM_FORMAT_MOD_NONE:
2931                 return 64;
2932         case I915_FORMAT_MOD_X_TILED:
2933                 if (INTEL_INFO(dev)->gen == 2)
2934                         return 128;
2935                 return 512;
2936         case I915_FORMAT_MOD_Y_TILED:
2937                 /* No need to check for old gens and Y tiling since this is
2938                  * about the display engine and those will be blocked before
2939                  * we get here.
2940                  */
2941                 return 128;
2942         case I915_FORMAT_MOD_Yf_TILED:
2943                 if (bits_per_pixel == 8)
2944                         return 64;
2945                 else
2946                         return 128;
2947         default:
2948                 MISSING_CASE(fb_modifier);
2949                 return 64;
2950         }
2951 }
2952
2953 unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
2954                                      struct drm_i915_gem_object *obj,
2955                                      unsigned int plane)
2956 {
2957         const struct i915_ggtt_view *view = &i915_ggtt_view_normal;
2958         struct i915_vma *vma;
2959         unsigned char *offset;
2960
2961         if (intel_rotation_90_or_270(intel_plane->base.state->rotation))
2962                 view = &i915_ggtt_view_rotated;
2963
2964         vma = i915_gem_obj_to_ggtt_view(obj, view);
2965         if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
2966                 view->type))
2967                 return -1;
2968
2969         offset = (unsigned char *)vma->node.start;
2970
2971         if (plane == 1) {
2972                 offset += vma->ggtt_view.rotation_info.uv_start_page *
2973                           PAGE_SIZE;
2974         }
2975
2976         return (unsigned long)offset;
2977 }
2978
2979 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
2980 {
2981         struct drm_device *dev = intel_crtc->base.dev;
2982         struct drm_i915_private *dev_priv = dev->dev_private;
2983
2984         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
2985         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
2986         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
2987 }
2988
2989 /*
2990  * This function detaches (aka. unbinds) unused scalers in hardware
2991  */
2992 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
2993 {
2994         struct intel_crtc_scaler_state *scaler_state;
2995         int i;
2996
2997         scaler_state = &intel_crtc->config->scaler_state;
2998
2999         /* loop through and disable scalers that aren't in use */
3000         for (i = 0; i < intel_crtc->num_scalers; i++) {
3001                 if (!scaler_state->scalers[i].in_use)
3002                         skl_detach_scaler(intel_crtc, i);
3003         }
3004 }
3005
3006 u32 skl_plane_ctl_format(uint32_t pixel_format)
3007 {
3008         switch (pixel_format) {
3009         case DRM_FORMAT_C8:
3010                 return PLANE_CTL_FORMAT_INDEXED;
3011         case DRM_FORMAT_RGB565:
3012                 return PLANE_CTL_FORMAT_RGB_565;
3013         case DRM_FORMAT_XBGR8888:
3014                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3015         case DRM_FORMAT_XRGB8888:
3016                 return PLANE_CTL_FORMAT_XRGB_8888;
3017         /*
3018          * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
3019          * to be already pre-multiplied. We need to add a knob (or a different
3020          * DRM_FORMAT) for user-space to configure that.
3021          */
3022         case DRM_FORMAT_ABGR8888:
3023                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
3024                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3025         case DRM_FORMAT_ARGB8888:
3026                 return PLANE_CTL_FORMAT_XRGB_8888 |
3027                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3028         case DRM_FORMAT_XRGB2101010:
3029                 return PLANE_CTL_FORMAT_XRGB_2101010;
3030         case DRM_FORMAT_XBGR2101010:
3031                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3032         case DRM_FORMAT_YUYV:
3033                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3034         case DRM_FORMAT_YVYU:
3035                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3036         case DRM_FORMAT_UYVY:
3037                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3038         case DRM_FORMAT_VYUY:
3039                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3040         default:
3041                 MISSING_CASE(pixel_format);
3042         }
3043
3044         return 0;
3045 }
3046
3047 u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3048 {
3049         switch (fb_modifier) {
3050         case DRM_FORMAT_MOD_NONE:
3051                 break;
3052         case I915_FORMAT_MOD_X_TILED:
3053                 return PLANE_CTL_TILED_X;
3054         case I915_FORMAT_MOD_Y_TILED:
3055                 return PLANE_CTL_TILED_Y;
3056         case I915_FORMAT_MOD_Yf_TILED:
3057                 return PLANE_CTL_TILED_YF;
3058         default:
3059                 MISSING_CASE(fb_modifier);
3060         }
3061
3062         return 0;
3063 }
3064
3065 u32 skl_plane_ctl_rotation(unsigned int rotation)
3066 {
3067         switch (rotation) {
3068         case BIT(DRM_ROTATE_0):
3069                 break;
3070         /*
3071          * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
3072          * while i915 HW rotation is clockwise, thats why this swapping.
3073          */
3074         case BIT(DRM_ROTATE_90):
3075                 return PLANE_CTL_ROTATE_270;
3076         case BIT(DRM_ROTATE_180):
3077                 return PLANE_CTL_ROTATE_180;
3078         case BIT(DRM_ROTATE_270):
3079                 return PLANE_CTL_ROTATE_90;
3080         default:
3081                 MISSING_CASE(rotation);
3082         }
3083
3084         return 0;
3085 }
3086
3087 static void skylake_update_primary_plane(struct drm_crtc *crtc,
3088                                          struct drm_framebuffer *fb,
3089                                          int x, int y)
3090 {
3091         struct drm_device *dev = crtc->dev;
3092         struct drm_i915_private *dev_priv = dev->dev_private;
3093         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3094         struct drm_plane *plane = crtc->primary;
3095         bool visible = to_intel_plane_state(plane->state)->visible;
3096         struct drm_i915_gem_object *obj;
3097         int pipe = intel_crtc->pipe;
3098         u32 plane_ctl, stride_div, stride;
3099         u32 tile_height, plane_offset, plane_size;
3100         unsigned int rotation;
3101         int x_offset, y_offset;
3102         unsigned long surf_addr;
3103         struct intel_crtc_state *crtc_state = intel_crtc->config;
3104         struct intel_plane_state *plane_state;
3105         int src_x = 0, src_y = 0, src_w = 0, src_h = 0;
3106         int dst_x = 0, dst_y = 0, dst_w = 0, dst_h = 0;
3107         int scaler_id = -1;
3108
3109         plane_state = to_intel_plane_state(plane->state);
3110
3111         if (!visible || !fb) {
3112                 I915_WRITE(PLANE_CTL(pipe, 0), 0);
3113                 I915_WRITE(PLANE_SURF(pipe, 0), 0);
3114                 POSTING_READ(PLANE_CTL(pipe, 0));
3115                 return;
3116         }
3117
3118         plane_ctl = PLANE_CTL_ENABLE |
3119                     PLANE_CTL_PIPE_GAMMA_ENABLE |
3120                     PLANE_CTL_PIPE_CSC_ENABLE;
3121
3122         plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
3123         plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
3124         plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
3125
3126         rotation = plane->state->rotation;
3127         plane_ctl |= skl_plane_ctl_rotation(rotation);
3128
3129         obj = intel_fb_obj(fb);
3130         stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
3131                                                fb->pixel_format);
3132         surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
3133
3134         WARN_ON(drm_rect_width(&plane_state->src) == 0);
3135
3136         scaler_id = plane_state->scaler_id;
3137         src_x = plane_state->src.x1 >> 16;
3138         src_y = plane_state->src.y1 >> 16;
3139         src_w = drm_rect_width(&plane_state->src) >> 16;
3140         src_h = drm_rect_height(&plane_state->src) >> 16;
3141         dst_x = plane_state->dst.x1;
3142         dst_y = plane_state->dst.y1;
3143         dst_w = drm_rect_width(&plane_state->dst);
3144         dst_h = drm_rect_height(&plane_state->dst);
3145
3146         WARN_ON(x != src_x || y != src_y);
3147
3148         if (intel_rotation_90_or_270(rotation)) {
3149                 /* stride = Surface height in tiles */
3150                 tile_height = intel_tile_height(dev, fb->pixel_format,
3151                                                 fb->modifier[0], 0);
3152                 stride = DIV_ROUND_UP(fb->height, tile_height);
3153                 x_offset = stride * tile_height - y - src_h;
3154                 y_offset = x;
3155                 plane_size = (src_w - 1) << 16 | (src_h - 1);
3156         } else {
3157                 stride = fb->pitches[0] / stride_div;
3158                 x_offset = x;
3159                 y_offset = y;
3160                 plane_size = (src_h - 1) << 16 | (src_w - 1);
3161         }
3162         plane_offset = y_offset << 16 | x_offset;
3163
3164         intel_crtc->adjusted_x = x_offset;
3165         intel_crtc->adjusted_y = y_offset;
3166
3167         I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
3168         I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
3169         I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
3170         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
3171
3172         if (scaler_id >= 0) {
3173                 uint32_t ps_ctrl = 0;
3174
3175                 WARN_ON(!dst_w || !dst_h);
3176                 ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(0) |
3177                         crtc_state->scaler_state.scalers[scaler_id].mode;
3178                 I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl);
3179                 I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0);
3180                 I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (dst_x << 16) | dst_y);
3181                 I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id), (dst_w << 16) | dst_h);
3182                 I915_WRITE(PLANE_POS(pipe, 0), 0);
3183         } else {
3184                 I915_WRITE(PLANE_POS(pipe, 0), (dst_y << 16) | dst_x);
3185         }
3186
3187         I915_WRITE(PLANE_SURF(pipe, 0), surf_addr);
3188
3189         POSTING_READ(PLANE_SURF(pipe, 0));
3190 }
3191
3192 /* Assume fb object is pinned & idle & fenced and just update base pointers */
3193 static int
3194 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
3195                            int x, int y, enum mode_set_atomic state)
3196 {
3197         struct drm_device *dev = crtc->dev;
3198         struct drm_i915_private *dev_priv = dev->dev_private;
3199
3200         if (dev_priv->fbc.disable_fbc)
3201                 dev_priv->fbc.disable_fbc(dev_priv);
3202
3203         dev_priv->display.update_primary_plane(crtc, fb, x, y);
3204
3205         return 0;
3206 }
3207
3208 static void intel_complete_page_flips(struct drm_device *dev)
3209 {
3210         struct drm_crtc *crtc;
3211
3212         for_each_crtc(dev, crtc) {
3213                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3214                 enum plane plane = intel_crtc->plane;
3215
3216                 intel_prepare_page_flip(dev, plane);
3217                 intel_finish_page_flip_plane(dev, plane);
3218         }
3219 }
3220
3221 static void intel_update_primary_planes(struct drm_device *dev)
3222 {
3223         struct drm_crtc *crtc;
3224
3225         for_each_crtc(dev, crtc) {
3226                 struct intel_plane *plane = to_intel_plane(crtc->primary);
3227                 struct intel_plane_state *plane_state;
3228
3229                 drm_modeset_lock_crtc(crtc, &plane->base);
3230
3231                 plane_state = to_intel_plane_state(plane->base.state);
3232
3233                 if (plane_state->base.fb)
3234                         plane->commit_plane(&plane->base, plane_state);
3235
3236                 drm_modeset_unlock_crtc(crtc);
3237         }
3238 }
3239
3240 void intel_prepare_reset(struct drm_device *dev)
3241 {
3242         /* no reset support for gen2 */
3243         if (IS_GEN2(dev))
3244                 return;
3245
3246         /* reset doesn't touch the display */
3247         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
3248                 return;
3249
3250         drm_modeset_lock_all(dev);
3251         /*
3252          * Disabling the crtcs gracefully seems nicer. Also the
3253          * g33 docs say we should at least disable all the planes.
3254          */
3255         intel_display_suspend(dev);
3256 }
3257
3258 void intel_finish_reset(struct drm_device *dev)
3259 {
3260         struct drm_i915_private *dev_priv = to_i915(dev);
3261
3262         /*
3263          * Flips in the rings will be nuked by the reset,
3264          * so complete all pending flips so that user space
3265          * will get its events and not get stuck.
3266          */
3267         intel_complete_page_flips(dev);
3268
3269         /* no reset support for gen2 */
3270         if (IS_GEN2(dev))
3271                 return;
3272
3273         /* reset doesn't touch the display */
3274         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) {
3275                 /*
3276                  * Flips in the rings have been nuked by the reset,
3277                  * so update the base address of all primary
3278                  * planes to the the last fb to make sure we're
3279                  * showing the correct fb after a reset.
3280                  *
3281                  * FIXME: Atomic will make this obsolete since we won't schedule
3282                  * CS-based flips (which might get lost in gpu resets) any more.
3283                  */
3284                 intel_update_primary_planes(dev);
3285                 return;
3286         }
3287
3288         /*
3289          * The display has been reset as well,
3290          * so need a full re-initialization.
3291          */
3292         intel_runtime_pm_disable_interrupts(dev_priv);
3293         intel_runtime_pm_enable_interrupts(dev_priv);
3294
3295         intel_modeset_init_hw(dev);
3296
3297         spin_lock_irq(&dev_priv->irq_lock);
3298         if (dev_priv->display.hpd_irq_setup)
3299                 dev_priv->display.hpd_irq_setup(dev);
3300         spin_unlock_irq(&dev_priv->irq_lock);
3301
3302         intel_display_resume(dev);
3303
3304         intel_hpd_init(dev_priv);
3305
3306         drm_modeset_unlock_all(dev);
3307 }
3308
3309 static void
3310 intel_finish_fb(struct drm_framebuffer *old_fb)
3311 {
3312         struct drm_i915_gem_object *obj = intel_fb_obj(old_fb);
3313         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3314         bool was_interruptible = dev_priv->mm.interruptible;
3315         int ret;
3316
3317         /* Big Hammer, we also need to ensure that any pending
3318          * MI_WAIT_FOR_EVENT inside a user batch buffer on the
3319          * current scanout is retired before unpinning the old
3320          * framebuffer. Note that we rely on userspace rendering
3321          * into the buffer attached to the pipe they are waiting
3322          * on. If not, userspace generates a GPU hang with IPEHR
3323          * point to the MI_WAIT_FOR_EVENT.
3324          *
3325          * This should only fail upon a hung GPU, in which case we
3326          * can safely continue.
3327          */
3328         dev_priv->mm.interruptible = false;
3329         ret = i915_gem_object_wait_rendering(obj, true);
3330         dev_priv->mm.interruptible = was_interruptible;
3331
3332         WARN_ON(ret);
3333 }
3334
3335 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
3336 {
3337         struct drm_device *dev = crtc->dev;
3338         struct drm_i915_private *dev_priv = dev->dev_private;
3339         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3340         bool pending;
3341
3342         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
3343             intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3344                 return false;
3345
3346         spin_lock_irq(&dev->event_lock);
3347         pending = to_intel_crtc(crtc)->unpin_work != NULL;
3348         spin_unlock_irq(&dev->event_lock);
3349
3350         return pending;
3351 }
3352
3353 static void intel_update_pipe_config(struct intel_crtc *crtc,
3354                                      struct intel_crtc_state *old_crtc_state)
3355 {
3356         struct drm_device *dev = crtc->base.dev;
3357         struct drm_i915_private *dev_priv = dev->dev_private;
3358         struct intel_crtc_state *pipe_config =
3359                 to_intel_crtc_state(crtc->base.state);
3360
3361         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3362         crtc->base.mode = crtc->base.state->mode;
3363
3364         DRM_DEBUG_KMS("Updating pipe size %ix%i -> %ix%i\n",
3365                       old_crtc_state->pipe_src_w, old_crtc_state->pipe_src_h,
3366                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
3367
3368         if (HAS_DDI(dev))
3369                 intel_set_pipe_csc(&crtc->base);
3370
3371         /*
3372          * Update pipe size and adjust fitter if needed: the reason for this is
3373          * that in compute_mode_changes we check the native mode (not the pfit
3374          * mode) to see if we can flip rather than do a full mode set. In the
3375          * fastboot case, we'll flip, but if we don't update the pipesrc and
3376          * pfit state, we'll end up with a big fb scanned out into the wrong
3377          * sized surface.
3378          */
3379
3380         I915_WRITE(PIPESRC(crtc->pipe),
3381                    ((pipe_config->pipe_src_w - 1) << 16) |
3382                    (pipe_config->pipe_src_h - 1));
3383
3384         /* on skylake this is done by detaching scalers */
3385         if (INTEL_INFO(dev)->gen >= 9) {
3386                 skl_detach_scalers(crtc);
3387
3388                 if (pipe_config->pch_pfit.enabled)
3389                         skylake_pfit_enable(crtc);
3390         } else if (HAS_PCH_SPLIT(dev)) {
3391                 if (pipe_config->pch_pfit.enabled)
3392                         ironlake_pfit_enable(crtc);
3393                 else if (old_crtc_state->pch_pfit.enabled)
3394                         ironlake_pfit_disable(crtc, true);
3395         }
3396 }
3397
3398 static void intel_fdi_normal_train(struct drm_crtc *crtc)
3399 {
3400         struct drm_device *dev = crtc->dev;
3401         struct drm_i915_private *dev_priv = dev->dev_private;
3402         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3403         int pipe = intel_crtc->pipe;
3404         u32 reg, temp;
3405
3406         /* enable normal train */
3407         reg = FDI_TX_CTL(pipe);
3408         temp = I915_READ(reg);
3409         if (IS_IVYBRIDGE(dev)) {
3410                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3411                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3412         } else {
3413                 temp &= ~FDI_LINK_TRAIN_NONE;
3414                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3415         }
3416         I915_WRITE(reg, temp);
3417
3418         reg = FDI_RX_CTL(pipe);
3419         temp = I915_READ(reg);
3420         if (HAS_PCH_CPT(dev)) {
3421                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3422                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3423         } else {
3424                 temp &= ~FDI_LINK_TRAIN_NONE;
3425                 temp |= FDI_LINK_TRAIN_NONE;
3426         }
3427         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3428
3429         /* wait one idle pattern time */
3430         POSTING_READ(reg);
3431         udelay(1000);
3432
3433         /* IVB wants error correction enabled */
3434         if (IS_IVYBRIDGE(dev))
3435                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3436                            FDI_FE_ERRC_ENABLE);
3437 }
3438
3439 /* The FDI link training functions for ILK/Ibexpeak. */
3440 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
3441 {
3442         struct drm_device *dev = crtc->dev;
3443         struct drm_i915_private *dev_priv = dev->dev_private;
3444         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3445         int pipe = intel_crtc->pipe;
3446         u32 reg, temp, tries;
3447
3448         /* FDI needs bits from pipe first */
3449         assert_pipe_enabled(dev_priv, pipe);
3450
3451         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3452            for train result */
3453         reg = FDI_RX_IMR(pipe);
3454         temp = I915_READ(reg);
3455         temp &= ~FDI_RX_SYMBOL_LOCK;
3456         temp &= ~FDI_RX_BIT_LOCK;
3457         I915_WRITE(reg, temp);
3458         I915_READ(reg);
3459         udelay(150);
3460
3461         /* enable CPU FDI TX and PCH FDI RX */
3462         reg = FDI_TX_CTL(pipe);
3463         temp = I915_READ(reg);
3464         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3465         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3466         temp &= ~FDI_LINK_TRAIN_NONE;
3467         temp |= FDI_LINK_TRAIN_PATTERN_1;
3468         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3469
3470         reg = FDI_RX_CTL(pipe);
3471         temp = I915_READ(reg);
3472         temp &= ~FDI_LINK_TRAIN_NONE;
3473         temp |= FDI_LINK_TRAIN_PATTERN_1;
3474         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3475
3476         POSTING_READ(reg);
3477         udelay(150);
3478
3479         /* Ironlake workaround, enable clock pointer after FDI enable*/
3480         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3481         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3482                    FDI_RX_PHASE_SYNC_POINTER_EN);
3483
3484         reg = FDI_RX_IIR(pipe);
3485         for (tries = 0; tries < 5; tries++) {
3486                 temp = I915_READ(reg);
3487                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3488
3489                 if ((temp & FDI_RX_BIT_LOCK)) {
3490                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3491                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3492                         break;
3493                 }
3494         }
3495         if (tries == 5)
3496                 DRM_ERROR("FDI train 1 fail!\n");
3497
3498         /* Train 2 */
3499         reg = FDI_TX_CTL(pipe);
3500         temp = I915_READ(reg);
3501         temp &= ~FDI_LINK_TRAIN_NONE;
3502         temp |= FDI_LINK_TRAIN_PATTERN_2;
3503         I915_WRITE(reg, temp);
3504
3505         reg = FDI_RX_CTL(pipe);
3506         temp = I915_READ(reg);
3507         temp &= ~FDI_LINK_TRAIN_NONE;
3508         temp |= FDI_LINK_TRAIN_PATTERN_2;
3509         I915_WRITE(reg, temp);
3510
3511         POSTING_READ(reg);
3512         udelay(150);
3513
3514         reg = FDI_RX_IIR(pipe);
3515         for (tries = 0; tries < 5; tries++) {
3516                 temp = I915_READ(reg);
3517                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3518
3519                 if (temp & FDI_RX_SYMBOL_LOCK) {
3520                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3521                         DRM_DEBUG_KMS("FDI train 2 done.\n");
3522                         break;
3523                 }
3524         }
3525         if (tries == 5)
3526                 DRM_ERROR("FDI train 2 fail!\n");
3527
3528         DRM_DEBUG_KMS("FDI train done\n");
3529
3530 }
3531
3532 static const int snb_b_fdi_train_param[] = {
3533         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
3534         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
3535         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
3536         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
3537 };
3538
3539 /* The FDI link training functions for SNB/Cougarpoint. */
3540 static void gen6_fdi_link_train(struct drm_crtc *crtc)
3541 {
3542         struct drm_device *dev = crtc->dev;
3543         struct drm_i915_private *dev_priv = dev->dev_private;
3544         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3545         int pipe = intel_crtc->pipe;
3546         u32 reg, temp, i, retry;
3547
3548         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3549            for train result */
3550         reg = FDI_RX_IMR(pipe);
3551         temp = I915_READ(reg);
3552         temp &= ~FDI_RX_SYMBOL_LOCK;
3553         temp &= ~FDI_RX_BIT_LOCK;
3554         I915_WRITE(reg, temp);
3555
3556         POSTING_READ(reg);
3557         udelay(150);
3558
3559         /* enable CPU FDI TX and PCH FDI RX */
3560         reg = FDI_TX_CTL(pipe);
3561         temp = I915_READ(reg);
3562         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3563         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3564         temp &= ~FDI_LINK_TRAIN_NONE;
3565         temp |= FDI_LINK_TRAIN_PATTERN_1;
3566         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3567         /* SNB-B */
3568         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3569         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3570
3571         I915_WRITE(FDI_RX_MISC(pipe),
3572                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3573
3574         reg = FDI_RX_CTL(pipe);
3575         temp = I915_READ(reg);
3576         if (HAS_PCH_CPT(dev)) {
3577                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3578                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3579         } else {
3580                 temp &= ~FDI_LINK_TRAIN_NONE;
3581                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3582         }
3583         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3584
3585         POSTING_READ(reg);
3586         udelay(150);
3587
3588         for (i = 0; i < 4; i++) {
3589                 reg = FDI_TX_CTL(pipe);
3590                 temp = I915_READ(reg);
3591                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3592                 temp |= snb_b_fdi_train_param[i];
3593                 I915_WRITE(reg, temp);
3594
3595                 POSTING_READ(reg);
3596                 udelay(500);
3597
3598                 for (retry = 0; retry < 5; retry++) {
3599                         reg = FDI_RX_IIR(pipe);
3600                         temp = I915_READ(reg);
3601                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3602                         if (temp & FDI_RX_BIT_LOCK) {
3603                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3604                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
3605                                 break;
3606                         }
3607                         udelay(50);
3608                 }
3609                 if (retry < 5)
3610                         break;
3611         }
3612         if (i == 4)
3613                 DRM_ERROR("FDI train 1 fail!\n");
3614
3615         /* Train 2 */
3616         reg = FDI_TX_CTL(pipe);
3617         temp = I915_READ(reg);
3618         temp &= ~FDI_LINK_TRAIN_NONE;
3619         temp |= FDI_LINK_TRAIN_PATTERN_2;
3620         if (IS_GEN6(dev)) {
3621                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3622                 /* SNB-B */
3623                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3624         }
3625         I915_WRITE(reg, temp);
3626
3627         reg = FDI_RX_CTL(pipe);
3628         temp = I915_READ(reg);
3629         if (HAS_PCH_CPT(dev)) {
3630                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3631                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3632         } else {
3633                 temp &= ~FDI_LINK_TRAIN_NONE;
3634                 temp |= FDI_LINK_TRAIN_PATTERN_2;
3635         }
3636         I915_WRITE(reg, temp);
3637
3638         POSTING_READ(reg);
3639         udelay(150);
3640
3641         for (i = 0; i < 4; i++) {
3642                 reg = FDI_TX_CTL(pipe);
3643                 temp = I915_READ(reg);
3644                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3645                 temp |= snb_b_fdi_train_param[i];
3646                 I915_WRITE(reg, temp);
3647
3648                 POSTING_READ(reg);
3649                 udelay(500);
3650
3651                 for (retry = 0; retry < 5; retry++) {
3652                         reg = FDI_RX_IIR(pipe);
3653                         temp = I915_READ(reg);
3654                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3655                         if (temp & FDI_RX_SYMBOL_LOCK) {
3656                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3657                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
3658                                 break;
3659                         }
3660                         udelay(50);
3661                 }
3662                 if (retry < 5)
3663                         break;
3664         }
3665         if (i == 4)
3666                 DRM_ERROR("FDI train 2 fail!\n");
3667
3668         DRM_DEBUG_KMS("FDI train done.\n");
3669 }
3670
3671 /* Manual link training for Ivy Bridge A0 parts */
3672 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
3673 {
3674         struct drm_device *dev = crtc->dev;
3675         struct drm_i915_private *dev_priv = dev->dev_private;
3676         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3677         int pipe = intel_crtc->pipe;
3678         u32 reg, temp, i, j;
3679
3680         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3681            for train result */
3682         reg = FDI_RX_IMR(pipe);
3683         temp = I915_READ(reg);
3684         temp &= ~FDI_RX_SYMBOL_LOCK;
3685         temp &= ~FDI_RX_BIT_LOCK;
3686         I915_WRITE(reg, temp);
3687
3688         POSTING_READ(reg);
3689         udelay(150);
3690
3691         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
3692                       I915_READ(FDI_RX_IIR(pipe)));
3693
3694         /* Try each vswing and preemphasis setting twice before moving on */
3695         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
3696                 /* disable first in case we need to retry */
3697                 reg = FDI_TX_CTL(pipe);
3698                 temp = I915_READ(reg);
3699                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
3700                 temp &= ~FDI_TX_ENABLE;
3701                 I915_WRITE(reg, temp);
3702
3703                 reg = FDI_RX_CTL(pipe);
3704                 temp = I915_READ(reg);
3705                 temp &= ~FDI_LINK_TRAIN_AUTO;
3706                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3707                 temp &= ~FDI_RX_ENABLE;
3708                 I915_WRITE(reg, temp);
3709
3710                 /* enable CPU FDI TX and PCH FDI RX */
3711                 reg = FDI_TX_CTL(pipe);
3712                 temp = I915_READ(reg);
3713                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
3714                 temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3715                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
3716                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3717                 temp |= snb_b_fdi_train_param[j/2];
3718                 temp |= FDI_COMPOSITE_SYNC;
3719                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
3720
3721                 I915_WRITE(FDI_RX_MISC(pipe),
3722                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3723
3724                 reg = FDI_RX_CTL(pipe);
3725                 temp = I915_READ(reg);
3726                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3727                 temp |= FDI_COMPOSITE_SYNC;
3728                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
3729
3730                 POSTING_READ(reg);
3731                 udelay(1); /* should be 0.5us */
3732
3733                 for (i = 0; i < 4; i++) {
3734                         reg = FDI_RX_IIR(pipe);
3735                         temp = I915_READ(reg);
3736                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3737
3738                         if (temp & FDI_RX_BIT_LOCK ||
3739                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
3740                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3741                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
3742                                               i);
3743                                 break;
3744                         }
3745                         udelay(1); /* should be 0.5us */
3746                 }
3747                 if (i == 4) {
3748                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
3749                         continue;
3750                 }
3751
3752                 /* Train 2 */
3753                 reg = FDI_TX_CTL(pipe);
3754                 temp = I915_READ(reg);
3755                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3756                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
3757                 I915_WRITE(reg, temp);
3758
3759                 reg = FDI_RX_CTL(pipe);
3760                 temp = I915_READ(reg);
3761                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3762                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3763                 I915_WRITE(reg, temp);
3764
3765                 POSTING_READ(reg);
3766                 udelay(2); /* should be 1.5us */
3767
3768                 for (i = 0; i < 4; i++) {
3769                         reg = FDI_RX_IIR(pipe);
3770                         temp = I915_READ(reg);
3771                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3772
3773                         if (temp & FDI_RX_SYMBOL_LOCK ||
3774                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
3775                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3776                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
3777                                               i);
3778                                 goto train_done;
3779                         }
3780                         udelay(2); /* should be 1.5us */
3781                 }
3782                 if (i == 4)
3783                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
3784         }
3785
3786 train_done:
3787         DRM_DEBUG_KMS("FDI train done.\n");
3788 }
3789
3790 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
3791 {
3792         struct drm_device *dev = intel_crtc->base.dev;
3793         struct drm_i915_private *dev_priv = dev->dev_private;
3794         int pipe = intel_crtc->pipe;
3795         u32 reg, temp;
3796
3797
3798         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
3799         reg = FDI_RX_CTL(pipe);
3800         temp = I915_READ(reg);
3801         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
3802         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3803         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3804         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
3805
3806         POSTING_READ(reg);
3807         udelay(200);
3808
3809         /* Switch from Rawclk to PCDclk */
3810         temp = I915_READ(reg);
3811         I915_WRITE(reg, temp | FDI_PCDCLK);
3812
3813         POSTING_READ(reg);
3814         udelay(200);
3815
3816         /* Enable CPU FDI TX PLL, always on for Ironlake */
3817         reg = FDI_TX_CTL(pipe);
3818         temp = I915_READ(reg);
3819         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
3820                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
3821
3822                 POSTING_READ(reg);
3823                 udelay(100);
3824         }
3825 }
3826
3827 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
3828 {
3829         struct drm_device *dev = intel_crtc->base.dev;
3830         struct drm_i915_private *dev_priv = dev->dev_private;
3831         int pipe = intel_crtc->pipe;
3832         u32 reg, temp;
3833
3834         /* Switch from PCDclk to Rawclk */
3835         reg = FDI_RX_CTL(pipe);
3836         temp = I915_READ(reg);
3837         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3838
3839         /* Disable CPU FDI TX PLL */
3840         reg = FDI_TX_CTL(pipe);
3841         temp = I915_READ(reg);
3842         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3843
3844         POSTING_READ(reg);
3845         udelay(100);
3846
3847         reg = FDI_RX_CTL(pipe);
3848         temp = I915_READ(reg);
3849         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3850
3851         /* Wait for the clocks to turn off. */
3852         POSTING_READ(reg);
3853         udelay(100);
3854 }
3855
3856 static void ironlake_fdi_disable(struct drm_crtc *crtc)
3857 {
3858         struct drm_device *dev = crtc->dev;
3859         struct drm_i915_private *dev_priv = dev->dev_private;
3860         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3861         int pipe = intel_crtc->pipe;
3862         u32 reg, temp;
3863
3864         /* disable CPU FDI tx and PCH FDI rx */
3865         reg = FDI_TX_CTL(pipe);
3866         temp = I915_READ(reg);
3867         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
3868         POSTING_READ(reg);
3869
3870         reg = FDI_RX_CTL(pipe);
3871         temp = I915_READ(reg);
3872         temp &= ~(0x7 << 16);
3873         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3874         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
3875
3876         POSTING_READ(reg);
3877         udelay(100);
3878
3879         /* Ironlake workaround, disable clock pointer after downing FDI */
3880         if (HAS_PCH_IBX(dev))
3881                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3882
3883         /* still set train pattern 1 */
3884         reg = FDI_TX_CTL(pipe);
3885         temp = I915_READ(reg);
3886         temp &= ~FDI_LINK_TRAIN_NONE;
3887         temp |= FDI_LINK_TRAIN_PATTERN_1;
3888         I915_WRITE(reg, temp);
3889
3890         reg = FDI_RX_CTL(pipe);
3891         temp = I915_READ(reg);
3892         if (HAS_PCH_CPT(dev)) {
3893                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3894                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3895         } else {
3896                 temp &= ~FDI_LINK_TRAIN_NONE;
3897                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3898         }
3899         /* BPC in FDI rx is consistent with that in PIPECONF */
3900         temp &= ~(0x07 << 16);
3901         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3902         I915_WRITE(reg, temp);
3903
3904         POSTING_READ(reg);
3905         udelay(100);
3906 }
3907
3908 bool intel_has_pending_fb_unpin(struct drm_device *dev)
3909 {
3910         struct intel_crtc *crtc;
3911
3912         /* Note that we don't need to be called with mode_config.lock here
3913          * as our list of CRTC objects is static for the lifetime of the
3914          * device and so cannot disappear as we iterate. Similarly, we can
3915          * happily treat the predicates as racy, atomic checks as userspace
3916          * cannot claim and pin a new fb without at least acquring the
3917          * struct_mutex and so serialising with us.
3918          */
3919         for_each_intel_crtc(dev, crtc) {
3920                 if (atomic_read(&crtc->unpin_work_count) == 0)
3921                         continue;
3922
3923                 if (crtc->unpin_work)
3924                         intel_wait_for_vblank(dev, crtc->pipe);
3925
3926                 return true;
3927         }
3928
3929         return false;
3930 }
3931
3932 static void page_flip_completed(struct intel_crtc *intel_crtc)
3933 {
3934         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3935         struct intel_unpin_work *work = intel_crtc->unpin_work;
3936
3937         /* ensure that the unpin work is consistent wrt ->pending. */
3938         smp_rmb();
3939         intel_crtc->unpin_work = NULL;
3940
3941         if (work->event)
3942                 drm_send_vblank_event(intel_crtc->base.dev,
3943                                       intel_crtc->pipe,
3944                                       work->event);
3945
3946         drm_crtc_vblank_put(&intel_crtc->base);
3947
3948         wake_up_all(&dev_priv->pending_flip_queue);
3949         queue_work(dev_priv->wq, &work->work);
3950
3951         trace_i915_flip_complete(intel_crtc->plane,
3952                                  work->pending_flip_obj);
3953 }
3954
3955 void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
3956 {
3957         struct drm_device *dev = crtc->dev;
3958         struct drm_i915_private *dev_priv = dev->dev_private;
3959
3960         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
3961         if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
3962                                        !intel_crtc_has_pending_flip(crtc),
3963                                        60*HZ) == 0)) {
3964                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3965
3966                 spin_lock_irq(&dev->event_lock);
3967                 if (intel_crtc->unpin_work) {
3968                         WARN_ONCE(1, "Removing stuck page flip\n");
3969                         page_flip_completed(intel_crtc);
3970                 }
3971                 spin_unlock_irq(&dev->event_lock);
3972         }
3973
3974         if (crtc->primary->fb) {
3975                 mutex_lock(&dev->struct_mutex);
3976                 intel_finish_fb(crtc->primary->fb);
3977                 mutex_unlock(&dev->struct_mutex);
3978         }
3979 }
3980
3981 /* Program iCLKIP clock to the desired frequency */
3982 static void lpt_program_iclkip(struct drm_crtc *crtc)
3983 {
3984         struct drm_device *dev = crtc->dev;
3985         struct drm_i915_private *dev_priv = dev->dev_private;
3986         int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
3987         u32 divsel, phaseinc, auxdiv, phasedir = 0;
3988         u32 temp;
3989
3990         mutex_lock(&dev_priv->sb_lock);
3991
3992         /* It is necessary to ungate the pixclk gate prior to programming
3993          * the divisors, and gate it back when it is done.
3994          */
3995         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
3996
3997         /* Disable SSCCTL */
3998         intel_sbi_write(dev_priv, SBI_SSCCTL6,
3999                         intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK) |
4000                                 SBI_SSCCTL_DISABLE,
4001                         SBI_ICLK);
4002
4003         /* 20MHz is a corner case which is out of range for the 7-bit divisor */
4004         if (clock == 20000) {
4005                 auxdiv = 1;
4006                 divsel = 0x41;
4007                 phaseinc = 0x20;
4008         } else {
4009                 /* The iCLK virtual clock root frequency is in MHz,
4010                  * but the adjusted_mode->crtc_clock in in KHz. To get the
4011                  * divisors, it is necessary to divide one by another, so we
4012                  * convert the virtual clock precision to KHz here for higher
4013                  * precision.
4014                  */
4015                 u32 iclk_virtual_root_freq = 172800 * 1000;
4016                 u32 iclk_pi_range = 64;
4017                 u32 desired_divisor, msb_divisor_value, pi_value;
4018
4019                 desired_divisor = (iclk_virtual_root_freq / clock);
4020                 msb_divisor_value = desired_divisor / iclk_pi_range;
4021                 pi_value = desired_divisor % iclk_pi_range;
4022
4023                 auxdiv = 0;
4024                 divsel = msb_divisor_value - 2;
4025                 phaseinc = pi_value;
4026         }
4027
4028         /* This should not happen with any sane values */
4029         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4030                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4031         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4032                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4033
4034         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4035                         clock,
4036                         auxdiv,
4037                         divsel,
4038                         phasedir,
4039                         phaseinc);
4040
4041         /* Program SSCDIVINTPHASE6 */
4042         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4043         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4044         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4045         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4046         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4047         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4048         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4049         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4050
4051         /* Program SSCAUXDIV */
4052         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4053         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4054         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4055         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4056
4057         /* Enable modulator and associated divider */
4058         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4059         temp &= ~SBI_SSCCTL_DISABLE;
4060         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4061
4062         /* Wait for initialization time */
4063         udelay(24);
4064
4065         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4066
4067         mutex_unlock(&dev_priv->sb_lock);
4068 }
4069
4070 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
4071                                                 enum pipe pch_transcoder)
4072 {
4073         struct drm_device *dev = crtc->base.dev;
4074         struct drm_i915_private *dev_priv = dev->dev_private;
4075         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
4076
4077         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4078                    I915_READ(HTOTAL(cpu_transcoder)));
4079         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4080                    I915_READ(HBLANK(cpu_transcoder)));
4081         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4082                    I915_READ(HSYNC(cpu_transcoder)));
4083
4084         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4085                    I915_READ(VTOTAL(cpu_transcoder)));
4086         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4087                    I915_READ(VBLANK(cpu_transcoder)));
4088         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4089                    I915_READ(VSYNC(cpu_transcoder)));
4090         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4091                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4092 }
4093
4094 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4095 {
4096         struct drm_i915_private *dev_priv = dev->dev_private;
4097         uint32_t temp;
4098
4099         temp = I915_READ(SOUTH_CHICKEN1);
4100         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4101                 return;
4102
4103         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4104         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4105
4106         temp &= ~FDI_BC_BIFURCATION_SELECT;
4107         if (enable)
4108                 temp |= FDI_BC_BIFURCATION_SELECT;
4109
4110         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4111         I915_WRITE(SOUTH_CHICKEN1, temp);
4112         POSTING_READ(SOUTH_CHICKEN1);
4113 }
4114
4115 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4116 {
4117         struct drm_device *dev = intel_crtc->base.dev;
4118
4119         switch (intel_crtc->pipe) {
4120         case PIPE_A:
4121                 break;
4122         case PIPE_B:
4123                 if (intel_crtc->config->fdi_lanes > 2)
4124                         cpt_set_fdi_bc_bifurcation(dev, false);
4125                 else
4126                         cpt_set_fdi_bc_bifurcation(dev, true);
4127
4128                 break;
4129         case PIPE_C:
4130                 cpt_set_fdi_bc_bifurcation(dev, true);
4131
4132                 break;
4133         default:
4134                 BUG();
4135         }
4136 }
4137
4138 /*
4139  * Enable PCH resources required for PCH ports:
4140  *   - PCH PLLs
4141  *   - FDI training & RX/TX
4142  *   - update transcoder timings
4143  *   - DP transcoding bits
4144  *   - transcoder
4145  */
4146 static void ironlake_pch_enable(struct drm_crtc *crtc)
4147 {
4148         struct drm_device *dev = crtc->dev;
4149         struct drm_i915_private *dev_priv = dev->dev_private;
4150         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4151         int pipe = intel_crtc->pipe;
4152         u32 reg, temp;
4153
4154         assert_pch_transcoder_disabled(dev_priv, pipe);
4155
4156         if (IS_IVYBRIDGE(dev))
4157                 ivybridge_update_fdi_bc_bifurcation(intel_crtc);
4158
4159         /* Write the TU size bits before fdi link training, so that error
4160          * detection works. */
4161         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4162                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4163
4164         /* For PCH output, training FDI link */
4165         dev_priv->display.fdi_link_train(crtc);
4166
4167         /* We need to program the right clock selection before writing the pixel
4168          * mutliplier into the DPLL. */
4169         if (HAS_PCH_CPT(dev)) {
4170                 u32 sel;
4171
4172                 temp = I915_READ(PCH_DPLL_SEL);
4173                 temp |= TRANS_DPLL_ENABLE(pipe);
4174                 sel = TRANS_DPLLB_SEL(pipe);
4175                 if (intel_crtc->config->shared_dpll == DPLL_ID_PCH_PLL_B)
4176                         temp |= sel;
4177                 else
4178                         temp &= ~sel;
4179                 I915_WRITE(PCH_DPLL_SEL, temp);
4180         }
4181
4182         /* XXX: pch pll's can be enabled any time before we enable the PCH
4183          * transcoder, and we actually should do this to not upset any PCH
4184          * transcoder that already use the clock when we share it.
4185          *
4186          * Note that enable_shared_dpll tries to do the right thing, but
4187          * get_shared_dpll unconditionally resets the pll - we need that to have
4188          * the right LVDS enable sequence. */
4189         intel_enable_shared_dpll(intel_crtc);
4190
4191         /* set transcoder timing, panel must allow it */
4192         assert_panel_unlocked(dev_priv, pipe);
4193         ironlake_pch_transcoder_set_timings(intel_crtc, pipe);
4194
4195         intel_fdi_normal_train(crtc);
4196
4197         /* For PCH DP, enable TRANS_DP_CTL */
4198         if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
4199                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4200                 reg = TRANS_DP_CTL(pipe);
4201                 temp = I915_READ(reg);
4202                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4203                           TRANS_DP_SYNC_MASK |
4204                           TRANS_DP_BPC_MASK);
4205                 temp |= TRANS_DP_OUTPUT_ENABLE;
4206                 temp |= bpc << 9; /* same format but at 11:9 */
4207
4208                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
4209                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4210                 if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
4211                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4212
4213                 switch (intel_trans_dp_port_sel(crtc)) {
4214                 case PCH_DP_B:
4215                         temp |= TRANS_DP_PORT_SEL_B;
4216                         break;
4217                 case PCH_DP_C:
4218                         temp |= TRANS_DP_PORT_SEL_C;
4219                         break;
4220                 case PCH_DP_D:
4221                         temp |= TRANS_DP_PORT_SEL_D;
4222                         break;
4223                 default:
4224                         BUG();
4225                 }
4226
4227                 I915_WRITE(reg, temp);
4228         }
4229
4230         ironlake_enable_pch_transcoder(dev_priv, pipe);
4231 }
4232
4233 static void lpt_pch_enable(struct drm_crtc *crtc)
4234 {
4235         struct drm_device *dev = crtc->dev;
4236         struct drm_i915_private *dev_priv = dev->dev_private;
4237         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4238         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
4239
4240         assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
4241
4242         lpt_program_iclkip(crtc);
4243
4244         /* Set transcoder timing. */
4245         ironlake_pch_transcoder_set_timings(intel_crtc, PIPE_A);
4246
4247         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4248 }
4249
4250 struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc,
4251                                                 struct intel_crtc_state *crtc_state)
4252 {
4253         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
4254         struct intel_shared_dpll *pll;
4255         struct intel_shared_dpll_config *shared_dpll;
4256         enum intel_dpll_id i;
4257         int max = dev_priv->num_shared_dpll;
4258
4259         shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state);
4260
4261         if (HAS_PCH_IBX(dev_priv->dev)) {
4262                 /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
4263                 i = (enum intel_dpll_id) crtc->pipe;
4264                 pll = &dev_priv->shared_dplls[i];
4265
4266                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4267                               crtc->base.base.id, pll->name);
4268
4269                 WARN_ON(shared_dpll[i].crtc_mask);
4270
4271                 goto found;
4272         }
4273
4274         if (IS_BROXTON(dev_priv->dev)) {
4275                 /* PLL is attached to port in bxt */
4276                 struct intel_encoder *encoder;
4277                 struct intel_digital_port *intel_dig_port;
4278
4279                 encoder = intel_ddi_get_crtc_new_encoder(crtc_state);
4280                 if (WARN_ON(!encoder))
4281                         return NULL;
4282
4283                 intel_dig_port = enc_to_dig_port(&encoder->base);
4284                 /* 1:1 mapping between ports and PLLs */
4285                 i = (enum intel_dpll_id)intel_dig_port->port;
4286                 pll = &dev_priv->shared_dplls[i];
4287                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4288                         crtc->base.base.id, pll->name);
4289                 WARN_ON(shared_dpll[i].crtc_mask);
4290
4291                 goto found;
4292         } else if (INTEL_INFO(dev_priv)->gen < 9 && HAS_DDI(dev_priv))
4293                 /* Do not consider SPLL */
4294                 max = 2;
4295
4296         for (i = 0; i < max; i++) {
4297                 pll = &dev_priv->shared_dplls[i];
4298
4299                 /* Only want to check enabled timings first */
4300                 if (shared_dpll[i].crtc_mask == 0)
4301                         continue;
4302
4303                 if (memcmp(&crtc_state->dpll_hw_state,
4304                            &shared_dpll[i].hw_state,
4305                            sizeof(crtc_state->dpll_hw_state)) == 0) {
4306                         DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, ative %d)\n",
4307                                       crtc->base.base.id, pll->name,
4308                                       shared_dpll[i].crtc_mask,
4309                                       pll->active);
4310                         goto found;
4311                 }
4312         }
4313
4314         /* Ok no matching timings, maybe there's a free one? */
4315         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4316                 pll = &dev_priv->shared_dplls[i];
4317                 if (shared_dpll[i].crtc_mask == 0) {
4318                         DRM_DEBUG_KMS("CRTC:%d allocated %s\n",
4319                                       crtc->base.base.id, pll->name);
4320                         goto found;
4321                 }
4322         }
4323
4324         return NULL;
4325
4326 found:
4327         if (shared_dpll[i].crtc_mask == 0)
4328                 shared_dpll[i].hw_state =
4329                         crtc_state->dpll_hw_state;
4330
4331         crtc_state->shared_dpll = i;
4332         DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name,
4333                          pipe_name(crtc->pipe));
4334
4335         shared_dpll[i].crtc_mask |= 1 << crtc->pipe;
4336
4337         return pll;
4338 }
4339
4340 static void intel_shared_dpll_commit(struct drm_atomic_state *state)
4341 {
4342         struct drm_i915_private *dev_priv = to_i915(state->dev);
4343         struct intel_shared_dpll_config *shared_dpll;
4344         struct intel_shared_dpll *pll;
4345         enum intel_dpll_id i;
4346
4347         if (!to_intel_atomic_state(state)->dpll_set)
4348                 return;
4349
4350         shared_dpll = to_intel_atomic_state(state)->shared_dpll;
4351         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4352                 pll = &dev_priv->shared_dplls[i];
4353                 pll->config = shared_dpll[i];
4354         }
4355 }
4356
4357 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4358 {
4359         struct drm_i915_private *dev_priv = dev->dev_private;
4360         int dslreg = PIPEDSL(pipe);
4361         u32 temp;
4362
4363         temp = I915_READ(dslreg);
4364         udelay(500);
4365         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4366                 if (wait_for(I915_READ(dslreg) != temp, 5))
4367                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4368         }
4369 }
4370
4371 static int
4372 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4373                   unsigned scaler_user, int *scaler_id, unsigned int rotation,
4374                   int src_w, int src_h, int dst_w, int dst_h)
4375 {
4376         struct intel_crtc_scaler_state *scaler_state =
4377                 &crtc_state->scaler_state;
4378         struct intel_crtc *intel_crtc =
4379                 to_intel_crtc(crtc_state->base.crtc);
4380         int need_scaling;
4381
4382         need_scaling = intel_rotation_90_or_270(rotation) ?
4383                 (src_h != dst_w || src_w != dst_h):
4384                 (src_w != dst_w || src_h != dst_h);
4385
4386         /*
4387          * if plane is being disabled or scaler is no more required or force detach
4388          *  - free scaler binded to this plane/crtc
4389          *  - in order to do this, update crtc->scaler_usage
4390          *
4391          * Here scaler state in crtc_state is set free so that
4392          * scaler can be assigned to other user. Actual register
4393          * update to free the scaler is done in plane/panel-fit programming.
4394          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4395          */
4396         if (force_detach || !need_scaling) {
4397                 if (*scaler_id >= 0) {
4398                         scaler_state->scaler_users &= ~(1 << scaler_user);
4399                         scaler_state->scalers[*scaler_id].in_use = 0;
4400
4401                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4402                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4403                                 intel_crtc->pipe, scaler_user, *scaler_id,
4404                                 scaler_state->scaler_users);
4405                         *scaler_id = -1;
4406                 }
4407                 return 0;
4408         }
4409
4410         /* range checks */
4411         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4412                 dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4413
4414                 src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4415                 dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) {
4416                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4417                         "size is out of scaler range\n",
4418                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4419                 return -EINVAL;
4420         }
4421
4422         /* mark this plane as a scaler user in crtc_state */
4423         scaler_state->scaler_users |= (1 << scaler_user);
4424         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4425                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4426                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4427                 scaler_state->scaler_users);
4428
4429         return 0;
4430 }
4431
4432 /**
4433  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4434  *
4435  * @state: crtc's scaler state
4436  *
4437  * Return
4438  *     0 - scaler_usage updated successfully
4439  *    error - requested scaling cannot be supported or other error condition
4440  */
4441 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4442 {
4443         struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
4444         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4445
4446         DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n",
4447                       intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX);
4448
4449         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4450                 &state->scaler_state.scaler_id, DRM_ROTATE_0,
4451                 state->pipe_src_w, state->pipe_src_h,
4452                 adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
4453 }
4454
4455 /**
4456  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4457  *
4458  * @state: crtc's scaler state
4459  * @plane_state: atomic plane state to update
4460  *
4461  * Return
4462  *     0 - scaler_usage updated successfully
4463  *    error - requested scaling cannot be supported or other error condition
4464  */
4465 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4466                                    struct intel_plane_state *plane_state)
4467 {
4468
4469         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4470         struct intel_plane *intel_plane =
4471                 to_intel_plane(plane_state->base.plane);
4472         struct drm_framebuffer *fb = plane_state->base.fb;
4473         int ret;
4474
4475         bool force_detach = !fb || !plane_state->visible;
4476
4477         DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n",
4478                       intel_plane->base.base.id, intel_crtc->pipe,
4479                       drm_plane_index(&intel_plane->base));
4480
4481         ret = skl_update_scaler(crtc_state, force_detach,
4482                                 drm_plane_index(&intel_plane->base),
4483                                 &plane_state->scaler_id,
4484                                 plane_state->base.rotation,
4485                                 drm_rect_width(&plane_state->src) >> 16,
4486                                 drm_rect_height(&plane_state->src) >> 16,
4487                                 drm_rect_width(&plane_state->dst),
4488                                 drm_rect_height(&plane_state->dst));
4489
4490         if (ret || plane_state->scaler_id < 0)
4491                 return ret;
4492
4493         /* check colorkey */
4494         if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) {
4495                 DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed",
4496                               intel_plane->base.base.id);
4497                 return -EINVAL;
4498         }
4499
4500         /* Check src format */
4501         switch (fb->pixel_format) {
4502         case DRM_FORMAT_RGB565:
4503         case DRM_FORMAT_XBGR8888:
4504         case DRM_FORMAT_XRGB8888:
4505         case DRM_FORMAT_ABGR8888:
4506         case DRM_FORMAT_ARGB8888:
4507         case DRM_FORMAT_XRGB2101010:
4508         case DRM_FORMAT_XBGR2101010:
4509         case DRM_FORMAT_YUYV:
4510         case DRM_FORMAT_YVYU:
4511         case DRM_FORMAT_UYVY:
4512         case DRM_FORMAT_VYUY:
4513                 break;
4514         default:
4515                 DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n",
4516                         intel_plane->base.base.id, fb->base.id, fb->pixel_format);
4517                 return -EINVAL;
4518         }
4519
4520         return 0;
4521 }
4522
4523 static void skylake_scaler_disable(struct intel_crtc *crtc)
4524 {
4525         int i;
4526
4527         for (i = 0; i < crtc->num_scalers; i++)
4528                 skl_detach_scaler(crtc, i);
4529 }
4530
4531 static void skylake_pfit_enable(struct intel_crtc *crtc)
4532 {
4533         struct drm_device *dev = crtc->base.dev;
4534         struct drm_i915_private *dev_priv = dev->dev_private;
4535         int pipe = crtc->pipe;
4536         struct intel_crtc_scaler_state *scaler_state =
4537                 &crtc->config->scaler_state;
4538
4539         DRM_DEBUG_KMS("for crtc_state = %p\n", crtc->config);
4540
4541         if (crtc->config->pch_pfit.enabled) {
4542                 int id;
4543
4544                 if (WARN_ON(crtc->config->scaler_state.scaler_id < 0)) {
4545                         DRM_ERROR("Requesting pfit without getting a scaler first\n");
4546                         return;
4547                 }
4548
4549                 id = scaler_state->scaler_id;
4550                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
4551                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
4552                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos);
4553                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size);
4554
4555                 DRM_DEBUG_KMS("for crtc_state = %p scaler_id = %d\n", crtc->config, id);
4556         }
4557 }
4558
4559 static void ironlake_pfit_enable(struct intel_crtc *crtc)
4560 {
4561         struct drm_device *dev = crtc->base.dev;
4562         struct drm_i915_private *dev_priv = dev->dev_private;
4563         int pipe = crtc->pipe;
4564
4565         if (crtc->config->pch_pfit.enabled) {
4566                 /* Force use of hard-coded filter coefficients
4567                  * as some pre-programmed values are broken,
4568                  * e.g. x201.
4569                  */
4570                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
4571                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
4572                                                  PF_PIPE_SEL_IVB(pipe));
4573                 else
4574                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
4575                 I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos);
4576                 I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size);
4577         }
4578 }
4579
4580 void hsw_enable_ips(struct intel_crtc *crtc)
4581 {
4582         struct drm_device *dev = crtc->base.dev;
4583         struct drm_i915_private *dev_priv = dev->dev_private;
4584
4585         if (!crtc->config->ips_enabled)
4586                 return;
4587
4588         /* We can only enable IPS after we enable a plane and wait for a vblank */
4589         intel_wait_for_vblank(dev, crtc->pipe);
4590
4591         assert_plane_enabled(dev_priv, crtc->plane);
4592         if (IS_BROADWELL(dev)) {
4593                 mutex_lock(&dev_priv->rps.hw_lock);
4594                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
4595                 mutex_unlock(&dev_priv->rps.hw_lock);
4596                 /* Quoting Art Runyan: "its not safe to expect any particular
4597                  * value in IPS_CTL bit 31 after enabling IPS through the
4598                  * mailbox." Moreover, the mailbox may return a bogus state,
4599                  * so we need to just enable it and continue on.
4600                  */
4601         } else {
4602                 I915_WRITE(IPS_CTL, IPS_ENABLE);
4603                 /* The bit only becomes 1 in the next vblank, so this wait here
4604                  * is essentially intel_wait_for_vblank. If we don't have this
4605                  * and don't wait for vblanks until the end of crtc_enable, then
4606                  * the HW state readout code will complain that the expected
4607                  * IPS_CTL value is not the one we read. */
4608                 if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
4609                         DRM_ERROR("Timed out waiting for IPS enable\n");
4610         }
4611 }
4612
4613 void hsw_disable_ips(struct intel_crtc *crtc)
4614 {
4615         struct drm_device *dev = crtc->base.dev;
4616         struct drm_i915_private *dev_priv = dev->dev_private;
4617
4618         if (!crtc->config->ips_enabled)
4619                 return;
4620
4621         assert_plane_enabled(dev_priv, crtc->plane);
4622         if (IS_BROADWELL(dev)) {
4623                 mutex_lock(&dev_priv->rps.hw_lock);
4624                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
4625                 mutex_unlock(&dev_priv->rps.hw_lock);
4626                 /* wait for pcode to finish disabling IPS, which may take up to 42ms */
4627                 if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42))
4628                         DRM_ERROR("Timed out waiting for IPS disable\n");
4629         } else {
4630                 I915_WRITE(IPS_CTL, 0);
4631                 POSTING_READ(IPS_CTL);
4632         }
4633
4634         /* We need to wait for a vblank before we can disable the plane. */
4635         intel_wait_for_vblank(dev, crtc->pipe);
4636 }
4637
4638 /** Loads the palette/gamma unit for the CRTC with the prepared values */
4639 static void intel_crtc_load_lut(struct drm_crtc *crtc)
4640 {
4641         struct drm_device *dev = crtc->dev;
4642         struct drm_i915_private *dev_priv = dev->dev_private;
4643         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4644         enum pipe pipe = intel_crtc->pipe;
4645         int i;
4646         bool reenable_ips = false;
4647
4648         /* The clocks have to be on to load the palette. */
4649         if (!crtc->state->active)
4650                 return;
4651
4652         if (HAS_GMCH_DISPLAY(dev_priv->dev)) {
4653                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI))
4654                         assert_dsi_pll_enabled(dev_priv);
4655                 else
4656                         assert_pll_enabled(dev_priv, pipe);
4657         }
4658
4659         /* Workaround : Do not read or write the pipe palette/gamma data while
4660          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
4661          */
4662         if (IS_HASWELL(dev) && intel_crtc->config->ips_enabled &&
4663             ((I915_READ(GAMMA_MODE(pipe)) & GAMMA_MODE_MODE_MASK) ==
4664              GAMMA_MODE_MODE_SPLIT)) {
4665                 hsw_disable_ips(intel_crtc);
4666                 reenable_ips = true;
4667         }
4668
4669         for (i = 0; i < 256; i++) {
4670                 u32 palreg;
4671
4672                 if (HAS_GMCH_DISPLAY(dev))
4673                         palreg = PALETTE(pipe, i);
4674                 else
4675                         palreg = LGC_PALETTE(pipe, i);
4676
4677                 I915_WRITE(palreg,
4678                            (intel_crtc->lut_r[i] << 16) |
4679                            (intel_crtc->lut_g[i] << 8) |
4680                            intel_crtc->lut_b[i]);
4681         }
4682
4683         if (reenable_ips)
4684                 hsw_enable_ips(intel_crtc);
4685 }
4686
4687 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
4688 {
4689         if (intel_crtc->overlay) {
4690                 struct drm_device *dev = intel_crtc->base.dev;
4691                 struct drm_i915_private *dev_priv = dev->dev_private;
4692
4693                 mutex_lock(&dev->struct_mutex);
4694                 dev_priv->mm.interruptible = false;
4695                 (void) intel_overlay_switch_off(intel_crtc->overlay);
4696                 dev_priv->mm.interruptible = true;
4697                 mutex_unlock(&dev->struct_mutex);
4698         }
4699
4700         /* Let userspace switch the overlay on again. In most cases userspace
4701          * has to recompute where to put it anyway.
4702          */
4703 }
4704
4705 /**
4706  * intel_post_enable_primary - Perform operations after enabling primary plane
4707  * @crtc: the CRTC whose primary plane was just enabled
4708  *
4709  * Performs potentially sleeping operations that must be done after the primary
4710  * plane is enabled, such as updating FBC and IPS.  Note that this may be
4711  * called due to an explicit primary plane update, or due to an implicit
4712  * re-enable that is caused when a sprite plane is updated to no longer
4713  * completely hide the primary plane.
4714  */
4715 static void
4716 intel_post_enable_primary(struct drm_crtc *crtc)
4717 {
4718         struct drm_device *dev = crtc->dev;
4719         struct drm_i915_private *dev_priv = dev->dev_private;
4720         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4721         int pipe = intel_crtc->pipe;
4722
4723         /*
4724          * BDW signals flip done immediately if the plane
4725          * is disabled, even if the plane enable is already
4726          * armed to occur at the next vblank :(
4727          */
4728         if (IS_BROADWELL(dev))
4729                 intel_wait_for_vblank(dev, pipe);
4730
4731         /*
4732          * FIXME IPS should be fine as long as one plane is
4733          * enabled, but in practice it seems to have problems
4734          * when going from primary only to sprite only and vice
4735          * versa.
4736          */
4737         hsw_enable_ips(intel_crtc);
4738
4739         /*
4740          * Gen2 reports pipe underruns whenever all planes are disabled.
4741          * So don't enable underrun reporting before at least some planes
4742          * are enabled.
4743          * FIXME: Need to fix the logic to work when we turn off all planes
4744          * but leave the pipe running.
4745          */
4746         if (IS_GEN2(dev))
4747                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4748
4749         /* Underruns don't raise interrupts, so check manually. */
4750         if (HAS_GMCH_DISPLAY(dev))
4751                 i9xx_check_fifo_underruns(dev_priv);
4752 }
4753
4754 /**
4755  * intel_pre_disable_primary - Perform operations before disabling primary plane
4756  * @crtc: the CRTC whose primary plane is to be disabled
4757  *
4758  * Performs potentially sleeping operations that must be done before the
4759  * primary plane is disabled, such as updating FBC and IPS.  Note that this may
4760  * be called due to an explicit primary plane update, or due to an implicit
4761  * disable that is caused when a sprite plane completely hides the primary
4762  * plane.
4763  */
4764 static void
4765 intel_pre_disable_primary(struct drm_crtc *crtc)
4766 {
4767         struct drm_device *dev = crtc->dev;
4768         struct drm_i915_private *dev_priv = dev->dev_private;
4769         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4770         int pipe = intel_crtc->pipe;
4771
4772         /*
4773          * Gen2 reports pipe underruns whenever all planes are disabled.
4774          * So diasble underrun reporting before all the planes get disabled.
4775          * FIXME: Need to fix the logic to work when we turn off all planes
4776          * but leave the pipe running.
4777          */
4778         if (IS_GEN2(dev))
4779                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4780
4781         /*
4782          * Vblank time updates from the shadow to live plane control register
4783          * are blocked if the memory self-refresh mode is active at that
4784          * moment. So to make sure the plane gets truly disabled, disable
4785          * first the self-refresh mode. The self-refresh enable bit in turn
4786          * will be checked/applied by the HW only at the next frame start
4787          * event which is after the vblank start event, so we need to have a
4788          * wait-for-vblank between disabling the plane and the pipe.
4789          */
4790         if (HAS_GMCH_DISPLAY(dev)) {
4791                 intel_set_memory_cxsr(dev_priv, false);
4792                 dev_priv->wm.vlv.cxsr = false;
4793                 intel_wait_for_vblank(dev, pipe);
4794         }
4795
4796         /*
4797          * FIXME IPS should be fine as long as one plane is
4798          * enabled, but in practice it seems to have problems
4799          * when going from primary only to sprite only and vice
4800          * versa.
4801          */
4802         hsw_disable_ips(intel_crtc);
4803 }
4804
4805 static void intel_post_plane_update(struct intel_crtc *crtc)
4806 {
4807         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4808         struct drm_device *dev = crtc->base.dev;
4809         struct drm_i915_private *dev_priv = dev->dev_private;
4810         struct drm_plane *plane;
4811
4812         if (atomic->wait_vblank)
4813                 intel_wait_for_vblank(dev, crtc->pipe);
4814
4815         intel_frontbuffer_flip(dev, atomic->fb_bits);
4816
4817         if (atomic->disable_cxsr)
4818                 crtc->wm.cxsr_allowed = true;
4819
4820         if (crtc->atomic.update_wm_post)
4821                 intel_update_watermarks(&crtc->base);
4822
4823         if (atomic->update_fbc)
4824                 intel_fbc_update(dev_priv);
4825
4826         if (atomic->post_enable_primary)
4827                 intel_post_enable_primary(&crtc->base);
4828
4829         drm_for_each_plane_mask(plane, dev, atomic->update_sprite_watermarks)
4830                 intel_update_sprite_watermarks(plane, &crtc->base,
4831                                                0, 0, 0, false, false);
4832
4833         memset(atomic, 0, sizeof(*atomic));
4834 }
4835
4836 static void intel_pre_plane_update(struct intel_crtc *crtc)
4837 {
4838         struct drm_device *dev = crtc->base.dev;
4839         struct drm_i915_private *dev_priv = dev->dev_private;
4840         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4841         struct drm_plane *p;
4842
4843         /* Track fb's for any planes being disabled */
4844         drm_for_each_plane_mask(p, dev, atomic->disabled_planes) {
4845                 struct intel_plane *plane = to_intel_plane(p);
4846
4847                 mutex_lock(&dev->struct_mutex);
4848                 i915_gem_track_fb(intel_fb_obj(plane->base.fb), NULL,
4849                                   plane->frontbuffer_bit);
4850                 mutex_unlock(&dev->struct_mutex);
4851         }
4852
4853         if (atomic->wait_for_flips)
4854                 intel_crtc_wait_for_pending_flips(&crtc->base);
4855
4856         if (atomic->disable_fbc)
4857                 intel_fbc_disable_crtc(crtc);
4858
4859         if (crtc->atomic.disable_ips)
4860                 hsw_disable_ips(crtc);
4861
4862         if (atomic->pre_disable_primary)
4863                 intel_pre_disable_primary(&crtc->base);
4864
4865         if (atomic->disable_cxsr) {
4866                 crtc->wm.cxsr_allowed = false;
4867                 intel_set_memory_cxsr(dev_priv, false);
4868         }
4869 }
4870
4871 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
4872 {
4873         struct drm_device *dev = crtc->dev;
4874         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4875         struct drm_plane *p;
4876         int pipe = intel_crtc->pipe;
4877
4878         intel_crtc_dpms_overlay_disable(intel_crtc);
4879
4880         drm_for_each_plane_mask(p, dev, plane_mask)
4881                 to_intel_plane(p)->disable_plane(p, crtc);
4882
4883         /*
4884          * FIXME: Once we grow proper nuclear flip support out of this we need
4885          * to compute the mask of flip planes precisely. For the time being
4886          * consider this a flip to a NULL plane.
4887          */
4888         intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
4889 }
4890
4891 static void ironlake_crtc_enable(struct drm_crtc *crtc)
4892 {
4893         struct drm_device *dev = crtc->dev;
4894         struct drm_i915_private *dev_priv = dev->dev_private;
4895         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4896         struct intel_encoder *encoder;
4897         int pipe = intel_crtc->pipe;
4898
4899         if (WARN_ON(intel_crtc->active))
4900                 return;
4901
4902         if (intel_crtc->config->has_pch_encoder)
4903                 intel_prepare_shared_dpll(intel_crtc);
4904
4905         if (intel_crtc->config->has_dp_encoder)
4906                 intel_dp_set_m_n(intel_crtc, M1_N1);
4907
4908         intel_set_pipe_timings(intel_crtc);
4909
4910         if (intel_crtc->config->has_pch_encoder) {
4911                 intel_cpu_transcoder_set_m_n(intel_crtc,
4912                                      &intel_crtc->config->fdi_m_n, NULL);
4913         }
4914
4915         ironlake_set_pipeconf(crtc);
4916
4917         intel_crtc->active = true;
4918
4919         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4920         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
4921
4922         for_each_encoder_on_crtc(dev, crtc, encoder)
4923                 if (encoder->pre_enable)
4924                         encoder->pre_enable(encoder);
4925
4926         if (intel_crtc->config->has_pch_encoder) {
4927                 /* Note: FDI PLL enabling _must_ be done before we enable the
4928                  * cpu pipes, hence this is separate from all the other fdi/pch
4929                  * enabling. */
4930                 ironlake_fdi_pll_enable(intel_crtc);
4931         } else {
4932                 assert_fdi_tx_disabled(dev_priv, pipe);
4933                 assert_fdi_rx_disabled(dev_priv, pipe);
4934         }
4935
4936         ironlake_pfit_enable(intel_crtc);
4937
4938         /*
4939          * On ILK+ LUT must be loaded before the pipe is running but with
4940          * clocks enabled
4941          */
4942         intel_crtc_load_lut(crtc);
4943
4944         intel_update_watermarks(crtc);
4945         intel_enable_pipe(intel_crtc);
4946
4947         if (intel_crtc->config->has_pch_encoder)
4948                 ironlake_pch_enable(crtc);
4949
4950         assert_vblank_disabled(crtc);
4951         drm_crtc_vblank_on(crtc);
4952
4953         for_each_encoder_on_crtc(dev, crtc, encoder)
4954                 encoder->enable(encoder);
4955
4956         if (HAS_PCH_CPT(dev))
4957                 cpt_verify_modeset(dev, intel_crtc->pipe);
4958 }
4959
4960 /* IPS only exists on ULT machines and is tied to pipe A. */
4961 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
4962 {
4963         return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
4964 }
4965
4966 static void haswell_crtc_enable(struct drm_crtc *crtc)
4967 {
4968         struct drm_device *dev = crtc->dev;
4969         struct drm_i915_private *dev_priv = dev->dev_private;
4970         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4971         struct intel_encoder *encoder;
4972         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
4973         struct intel_crtc_state *pipe_config =
4974                 to_intel_crtc_state(crtc->state);
4975         bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
4976
4977         if (WARN_ON(intel_crtc->active))
4978                 return;
4979
4980         if (intel_crtc_to_shared_dpll(intel_crtc))
4981                 intel_enable_shared_dpll(intel_crtc);
4982
4983         if (intel_crtc->config->has_dp_encoder)
4984                 intel_dp_set_m_n(intel_crtc, M1_N1);
4985
4986         intel_set_pipe_timings(intel_crtc);
4987
4988         if (intel_crtc->config->cpu_transcoder != TRANSCODER_EDP) {
4989                 I915_WRITE(PIPE_MULT(intel_crtc->config->cpu_transcoder),
4990                            intel_crtc->config->pixel_multiplier - 1);
4991         }
4992
4993         if (intel_crtc->config->has_pch_encoder) {
4994                 intel_cpu_transcoder_set_m_n(intel_crtc,
4995                                      &intel_crtc->config->fdi_m_n, NULL);
4996         }
4997
4998         haswell_set_pipeconf(crtc);
4999
5000         intel_set_pipe_csc(crtc);
5001
5002         intel_crtc->active = true;
5003
5004         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5005         for_each_encoder_on_crtc(dev, crtc, encoder) {
5006                 if (encoder->pre_pll_enable)
5007                         encoder->pre_pll_enable(encoder);
5008                 if (encoder->pre_enable)
5009                         encoder->pre_enable(encoder);
5010         }
5011
5012         if (intel_crtc->config->has_pch_encoder) {
5013                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5014                                                       true);
5015                 dev_priv->display.fdi_link_train(crtc);
5016         }
5017
5018         if (!is_dsi)
5019                 intel_ddi_enable_pipe_clock(intel_crtc);
5020
5021         if (INTEL_INFO(dev)->gen >= 9)
5022                 skylake_pfit_enable(intel_crtc);
5023         else
5024                 ironlake_pfit_enable(intel_crtc);
5025
5026         /*
5027          * On ILK+ LUT must be loaded before the pipe is running but with
5028          * clocks enabled
5029          */
5030         intel_crtc_load_lut(crtc);
5031
5032         intel_ddi_set_pipe_settings(crtc);
5033         if (!is_dsi)
5034                 intel_ddi_enable_transcoder_func(crtc);
5035
5036         intel_update_watermarks(crtc);
5037         intel_enable_pipe(intel_crtc);
5038
5039         if (intel_crtc->config->has_pch_encoder)
5040                 lpt_pch_enable(crtc);
5041
5042         if (intel_crtc->config->dp_encoder_is_mst && !is_dsi)
5043                 intel_ddi_set_vc_payload_alloc(crtc, true);
5044
5045         assert_vblank_disabled(crtc);
5046         drm_crtc_vblank_on(crtc);
5047
5048         for_each_encoder_on_crtc(dev, crtc, encoder) {
5049                 encoder->enable(encoder);
5050                 intel_opregion_notify_encoder(encoder, true);
5051         }
5052
5053         /* If we change the relative order between pipe/planes enabling, we need
5054          * to change the workaround. */
5055         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5056         if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
5057                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
5058                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
5059         }
5060 }
5061
5062 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
5063 {
5064         struct drm_device *dev = crtc->base.dev;
5065         struct drm_i915_private *dev_priv = dev->dev_private;
5066         int pipe = crtc->pipe;
5067
5068         /* To avoid upsetting the power well on haswell only disable the pfit if
5069          * it's in use. The hw state code will make sure we get this right. */
5070         if (force || crtc->config->pch_pfit.enabled) {
5071                 I915_WRITE(PF_CTL(pipe), 0);
5072                 I915_WRITE(PF_WIN_POS(pipe), 0);
5073                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5074         }
5075 }
5076
5077 static void ironlake_crtc_disable(struct drm_crtc *crtc)
5078 {
5079         struct drm_device *dev = crtc->dev;
5080         struct drm_i915_private *dev_priv = dev->dev_private;
5081         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5082         struct intel_encoder *encoder;
5083         int pipe = intel_crtc->pipe;
5084         u32 reg, temp;
5085
5086         for_each_encoder_on_crtc(dev, crtc, encoder)
5087                 encoder->disable(encoder);
5088
5089         drm_crtc_vblank_off(crtc);
5090         assert_vblank_disabled(crtc);
5091
5092         if (intel_crtc->config->has_pch_encoder)
5093                 intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5094
5095         intel_disable_pipe(intel_crtc);
5096
5097         ironlake_pfit_disable(intel_crtc, false);
5098
5099         if (intel_crtc->config->has_pch_encoder)
5100                 ironlake_fdi_disable(crtc);
5101
5102         for_each_encoder_on_crtc(dev, crtc, encoder)
5103                 if (encoder->post_disable)
5104                         encoder->post_disable(encoder);
5105
5106         if (intel_crtc->config->has_pch_encoder) {
5107                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5108
5109                 if (HAS_PCH_CPT(dev)) {
5110                         /* disable TRANS_DP_CTL */
5111                         reg = TRANS_DP_CTL(pipe);
5112                         temp = I915_READ(reg);
5113                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5114                                   TRANS_DP_PORT_SEL_MASK);
5115                         temp |= TRANS_DP_PORT_SEL_NONE;
5116                         I915_WRITE(reg, temp);
5117
5118                         /* disable DPLL_SEL */
5119                         temp = I915_READ(PCH_DPLL_SEL);
5120                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5121                         I915_WRITE(PCH_DPLL_SEL, temp);
5122                 }
5123
5124                 ironlake_fdi_pll_disable(intel_crtc);
5125         }
5126 }
5127
5128 static void haswell_crtc_disable(struct drm_crtc *crtc)
5129 {
5130         struct drm_device *dev = crtc->dev;
5131         struct drm_i915_private *dev_priv = dev->dev_private;
5132         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5133         struct intel_encoder *encoder;
5134         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5135         bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
5136
5137         for_each_encoder_on_crtc(dev, crtc, encoder) {
5138                 intel_opregion_notify_encoder(encoder, false);
5139                 encoder->disable(encoder);
5140         }
5141
5142         drm_crtc_vblank_off(crtc);
5143         assert_vblank_disabled(crtc);
5144
5145         if (intel_crtc->config->has_pch_encoder)
5146                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5147                                                       false);
5148         intel_disable_pipe(intel_crtc);
5149
5150         if (intel_crtc->config->dp_encoder_is_mst)
5151                 intel_ddi_set_vc_payload_alloc(crtc, false);
5152
5153         if (!is_dsi)
5154                 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
5155
5156         if (INTEL_INFO(dev)->gen >= 9)
5157                 skylake_scaler_disable(intel_crtc);
5158         else
5159                 ironlake_pfit_disable(intel_crtc, false);
5160
5161         if (!is_dsi)
5162                 intel_ddi_disable_pipe_clock(intel_crtc);
5163
5164         if (intel_crtc->config->has_pch_encoder) {
5165                 lpt_disable_pch_transcoder(dev_priv);
5166                 intel_ddi_fdi_disable(crtc);
5167         }
5168
5169         for_each_encoder_on_crtc(dev, crtc, encoder)
5170                 if (encoder->post_disable)
5171                         encoder->post_disable(encoder);
5172 }
5173
5174 static void i9xx_pfit_enable(struct intel_crtc *crtc)
5175 {
5176         struct drm_device *dev = crtc->base.dev;
5177         struct drm_i915_private *dev_priv = dev->dev_private;
5178         struct intel_crtc_state *pipe_config = crtc->config;
5179
5180         if (!pipe_config->gmch_pfit.control)
5181                 return;
5182
5183         /*
5184          * The panel fitter should only be adjusted whilst the pipe is disabled,
5185          * according to register description and PRM.
5186          */
5187         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5188         assert_pipe_disabled(dev_priv, crtc->pipe);
5189
5190         I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
5191         I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
5192
5193         /* Border color in case we don't scale up to the full screen. Black by
5194          * default, change to something else for debugging. */
5195         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5196 }
5197
5198 static enum intel_display_power_domain port_to_power_domain(enum port port)
5199 {
5200         switch (port) {
5201         case PORT_A:
5202                 return POWER_DOMAIN_PORT_DDI_A_4_LANES;
5203         case PORT_B:
5204                 return POWER_DOMAIN_PORT_DDI_B_4_LANES;
5205         case PORT_C:
5206                 return POWER_DOMAIN_PORT_DDI_C_4_LANES;
5207         case PORT_D:
5208                 return POWER_DOMAIN_PORT_DDI_D_4_LANES;
5209         case PORT_E:
5210                 return POWER_DOMAIN_PORT_DDI_E_2_LANES;
5211         default:
5212                 MISSING_CASE(port);
5213                 return POWER_DOMAIN_PORT_OTHER;
5214         }
5215 }
5216
5217 static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
5218 {
5219         switch (port) {
5220         case PORT_A:
5221                 return POWER_DOMAIN_AUX_A;
5222         case PORT_B:
5223                 return POWER_DOMAIN_AUX_B;
5224         case PORT_C:
5225                 return POWER_DOMAIN_AUX_C;
5226         case PORT_D:
5227                 return POWER_DOMAIN_AUX_D;
5228         case PORT_E:
5229                 /* FIXME: Check VBT for actual wiring of PORT E */
5230                 return POWER_DOMAIN_AUX_D;
5231         default:
5232                 MISSING_CASE(port);
5233                 return POWER_DOMAIN_AUX_A;
5234         }
5235 }
5236
5237 #define for_each_power_domain(domain, mask)                             \
5238         for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
5239                 if ((1 << (domain)) & (mask))
5240
5241 enum intel_display_power_domain
5242 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
5243 {
5244         struct drm_device *dev = intel_encoder->base.dev;
5245         struct intel_digital_port *intel_dig_port;
5246
5247         switch (intel_encoder->type) {
5248         case INTEL_OUTPUT_UNKNOWN:
5249                 /* Only DDI platforms should ever use this output type */
5250                 WARN_ON_ONCE(!HAS_DDI(dev));
5251         case INTEL_OUTPUT_DISPLAYPORT:
5252         case INTEL_OUTPUT_HDMI:
5253         case INTEL_OUTPUT_EDP:
5254                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5255                 return port_to_power_domain(intel_dig_port->port);
5256         case INTEL_OUTPUT_DP_MST:
5257                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5258                 return port_to_power_domain(intel_dig_port->port);
5259         case INTEL_OUTPUT_ANALOG:
5260                 return POWER_DOMAIN_PORT_CRT;
5261         case INTEL_OUTPUT_DSI:
5262                 return POWER_DOMAIN_PORT_DSI;
5263         default:
5264                 return POWER_DOMAIN_PORT_OTHER;
5265         }
5266 }
5267
5268 enum intel_display_power_domain
5269 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
5270 {
5271         struct drm_device *dev = intel_encoder->base.dev;
5272         struct intel_digital_port *intel_dig_port;
5273
5274         switch (intel_encoder->type) {
5275         case INTEL_OUTPUT_UNKNOWN:
5276         case INTEL_OUTPUT_HDMI:
5277                 /*
5278                  * Only DDI platforms should ever use these output types.
5279                  * We can get here after the HDMI detect code has already set
5280                  * the type of the shared encoder. Since we can't be sure
5281                  * what's the status of the given connectors, play safe and
5282                  * run the DP detection too.
5283                  */
5284                 WARN_ON_ONCE(!HAS_DDI(dev));
5285         case INTEL_OUTPUT_DISPLAYPORT:
5286         case INTEL_OUTPUT_EDP:
5287                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5288                 return port_to_aux_power_domain(intel_dig_port->port);
5289         case INTEL_OUTPUT_DP_MST:
5290                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5291                 return port_to_aux_power_domain(intel_dig_port->port);
5292         default:
5293                 MISSING_CASE(intel_encoder->type);
5294                 return POWER_DOMAIN_AUX_A;
5295         }
5296 }
5297
5298 static unsigned long get_crtc_power_domains(struct drm_crtc *crtc)
5299 {
5300         struct drm_device *dev = crtc->dev;
5301         struct intel_encoder *intel_encoder;
5302         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5303         enum pipe pipe = intel_crtc->pipe;
5304         unsigned long mask;
5305         enum transcoder transcoder;
5306
5307         if (!crtc->state->active)
5308                 return 0;
5309
5310         transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
5311
5312         mask = BIT(POWER_DOMAIN_PIPE(pipe));
5313         mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
5314         if (intel_crtc->config->pch_pfit.enabled ||
5315             intel_crtc->config->pch_pfit.force_thru)
5316                 mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
5317
5318         for_each_encoder_on_crtc(dev, crtc, intel_encoder)
5319                 mask |= BIT(intel_display_port_power_domain(intel_encoder));
5320
5321         return mask;
5322 }
5323
5324 static unsigned long modeset_get_crtc_power_domains(struct drm_crtc *crtc)
5325 {
5326         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
5327         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5328         enum intel_display_power_domain domain;
5329         unsigned long domains, new_domains, old_domains;
5330
5331         old_domains = intel_crtc->enabled_power_domains;
5332         intel_crtc->enabled_power_domains = new_domains = get_crtc_power_domains(crtc);
5333
5334         domains = new_domains & ~old_domains;
5335
5336         for_each_power_domain(domain, domains)
5337                 intel_display_power_get(dev_priv, domain);
5338
5339         return old_domains & ~new_domains;
5340 }
5341
5342 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
5343                                       unsigned long domains)
5344 {
5345         enum intel_display_power_domain domain;
5346
5347         for_each_power_domain(domain, domains)
5348                 intel_display_power_put(dev_priv, domain);
5349 }
5350
5351 static void modeset_update_crtc_power_domains(struct drm_atomic_state *state)
5352 {
5353         struct drm_device *dev = state->dev;
5354         struct drm_i915_private *dev_priv = dev->dev_private;
5355         unsigned long put_domains[I915_MAX_PIPES] = {};
5356         struct drm_crtc_state *crtc_state;
5357         struct drm_crtc *crtc;
5358         int i;
5359
5360         for_each_crtc_in_state(state, crtc, crtc_state, i) {
5361                 if (needs_modeset(crtc->state))
5362                         put_domains[to_intel_crtc(crtc)->pipe] =
5363                                 modeset_get_crtc_power_domains(crtc);
5364         }
5365
5366         if (dev_priv->display.modeset_commit_cdclk) {
5367                 unsigned int cdclk = to_intel_atomic_state(state)->cdclk;
5368
5369                 if (cdclk != dev_priv->cdclk_freq &&
5370                     !WARN_ON(!state->allow_modeset))
5371                         dev_priv->display.modeset_commit_cdclk(state);
5372         }
5373
5374         for (i = 0; i < I915_MAX_PIPES; i++)
5375                 if (put_domains[i])
5376                         modeset_put_power_domains(dev_priv, put_domains[i]);
5377 }
5378
5379 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
5380 {
5381         int max_cdclk_freq = dev_priv->max_cdclk_freq;
5382
5383         if (INTEL_INFO(dev_priv)->gen >= 9 ||
5384             IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5385                 return max_cdclk_freq;
5386         else if (IS_CHERRYVIEW(dev_priv))
5387                 return max_cdclk_freq*95/100;
5388         else if (INTEL_INFO(dev_priv)->gen < 4)
5389                 return 2*max_cdclk_freq*90/100;
5390         else
5391                 return max_cdclk_freq*90/100;
5392 }
5393
5394 static void intel_update_max_cdclk(struct drm_device *dev)
5395 {
5396         struct drm_i915_private *dev_priv = dev->dev_private;
5397
5398         if (IS_SKYLAKE(dev)) {
5399                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
5400
5401                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
5402                         dev_priv->max_cdclk_freq = 675000;
5403                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
5404                         dev_priv->max_cdclk_freq = 540000;
5405                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
5406                         dev_priv->max_cdclk_freq = 450000;
5407                 else
5408                         dev_priv->max_cdclk_freq = 337500;
5409         } else if (IS_BROADWELL(dev))  {
5410                 /*
5411                  * FIXME with extra cooling we can allow
5412                  * 540 MHz for ULX and 675 Mhz for ULT.
5413                  * How can we know if extra cooling is
5414                  * available? PCI ID, VTB, something else?
5415                  */
5416                 if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
5417                         dev_priv->max_cdclk_freq = 450000;
5418                 else if (IS_BDW_ULX(dev))
5419                         dev_priv->max_cdclk_freq = 450000;
5420                 else if (IS_BDW_ULT(dev))
5421                         dev_priv->max_cdclk_freq = 540000;
5422                 else
5423                         dev_priv->max_cdclk_freq = 675000;
5424         } else if (IS_CHERRYVIEW(dev)) {
5425                 dev_priv->max_cdclk_freq = 320000;
5426         } else if (IS_VALLEYVIEW(dev)) {
5427                 dev_priv->max_cdclk_freq = 400000;
5428         } else {
5429                 /* otherwise assume cdclk is fixed */
5430                 dev_priv->max_cdclk_freq = dev_priv->cdclk_freq;
5431         }
5432
5433         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
5434
5435         DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n",
5436                          dev_priv->max_cdclk_freq);
5437
5438         DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n",
5439                          dev_priv->max_dotclk_freq);
5440 }
5441
5442 static void intel_update_cdclk(struct drm_device *dev)
5443 {
5444         struct drm_i915_private *dev_priv = dev->dev_private;
5445
5446         dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
5447         DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
5448                          dev_priv->cdclk_freq);
5449
5450         /*
5451          * Program the gmbus_freq based on the cdclk frequency.
5452          * BSpec erroneously claims we should aim for 4MHz, but
5453          * in fact 1MHz is the correct frequency.
5454          */
5455         if (IS_VALLEYVIEW(dev)) {
5456                 /*
5457                  * Program the gmbus_freq based on the cdclk frequency.
5458                  * BSpec erroneously claims we should aim for 4MHz, but
5459                  * in fact 1MHz is the correct frequency.
5460                  */
5461                 I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000));
5462         }
5463
5464         if (dev_priv->max_cdclk_freq == 0)
5465                 intel_update_max_cdclk(dev);
5466 }
5467
5468 static void broxton_set_cdclk(struct drm_device *dev, int frequency)
5469 {
5470         struct drm_i915_private *dev_priv = dev->dev_private;
5471         uint32_t divider;
5472         uint32_t ratio;
5473         uint32_t current_freq;
5474         int ret;
5475
5476         /* frequency = 19.2MHz * ratio / 2 / div{1,1.5,2,4} */
5477         switch (frequency) {
5478         case 144000:
5479                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
5480                 ratio = BXT_DE_PLL_RATIO(60);
5481                 break;
5482         case 288000:
5483                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
5484                 ratio = BXT_DE_PLL_RATIO(60);
5485                 break;
5486         case 384000:
5487                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
5488                 ratio = BXT_DE_PLL_RATIO(60);
5489                 break;
5490         case 576000:
5491                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5492                 ratio = BXT_DE_PLL_RATIO(60);
5493                 break;
5494         case 624000:
5495                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5496                 ratio = BXT_DE_PLL_RATIO(65);
5497                 break;
5498         case 19200:
5499                 /*
5500                  * Bypass frequency with DE PLL disabled. Init ratio, divider
5501                  * to suppress GCC warning.
5502                  */
5503                 ratio = 0;
5504                 divider = 0;
5505                 break;
5506         default:
5507                 DRM_ERROR("unsupported CDCLK freq %d", frequency);
5508
5509                 return;
5510         }
5511
5512         mutex_lock(&dev_priv->rps.hw_lock);
5513         /* Inform power controller of upcoming frequency change */
5514         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5515                                       0x80000000);
5516         mutex_unlock(&dev_priv->rps.hw_lock);
5517
5518         if (ret) {
5519                 DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n",
5520                           ret, frequency);
5521                 return;
5522         }
5523
5524         current_freq = I915_READ(CDCLK_CTL) & CDCLK_FREQ_DECIMAL_MASK;
5525         /* convert from .1 fixpoint MHz with -1MHz offset to kHz */
5526         current_freq = current_freq * 500 + 1000;
5527
5528         /*
5529          * DE PLL has to be disabled when
5530          * - setting to 19.2MHz (bypass, PLL isn't used)
5531          * - before setting to 624MHz (PLL needs toggling)
5532          * - before setting to any frequency from 624MHz (PLL needs toggling)
5533          */
5534         if (frequency == 19200 || frequency == 624000 ||
5535             current_freq == 624000) {
5536                 I915_WRITE(BXT_DE_PLL_ENABLE, ~BXT_DE_PLL_PLL_ENABLE);
5537                 /* Timeout 200us */
5538                 if (wait_for(!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK),
5539                              1))
5540                         DRM_ERROR("timout waiting for DE PLL unlock\n");
5541         }
5542
5543         if (frequency != 19200) {
5544                 uint32_t val;
5545
5546                 val = I915_READ(BXT_DE_PLL_CTL);
5547                 val &= ~BXT_DE_PLL_RATIO_MASK;
5548                 val |= ratio;
5549                 I915_WRITE(BXT_DE_PLL_CTL, val);
5550
5551                 I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
5552                 /* Timeout 200us */
5553                 if (wait_for(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK, 1))
5554                         DRM_ERROR("timeout waiting for DE PLL lock\n");
5555
5556                 val = I915_READ(CDCLK_CTL);
5557                 val &= ~BXT_CDCLK_CD2X_DIV_SEL_MASK;
5558                 val |= divider;
5559                 /*
5560                  * Disable SSA Precharge when CD clock frequency < 500 MHz,
5561                  * enable otherwise.
5562                  */
5563                 val &= ~BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5564                 if (frequency >= 500000)
5565                         val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5566
5567                 val &= ~CDCLK_FREQ_DECIMAL_MASK;
5568                 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
5569                 val |= (frequency - 1000) / 500;
5570                 I915_WRITE(CDCLK_CTL, val);
5571         }
5572
5573         mutex_lock(&dev_priv->rps.hw_lock);
5574         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5575                                       DIV_ROUND_UP(frequency, 25000));
5576         mutex_unlock(&dev_priv->rps.hw_lock);
5577
5578         if (ret) {
5579                 DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n",
5580                           ret, frequency);
5581                 return;
5582         }
5583
5584         intel_update_cdclk(dev);
5585 }
5586
5587 void broxton_init_cdclk(struct drm_device *dev)
5588 {
5589         struct drm_i915_private *dev_priv = dev->dev_private;
5590         uint32_t val;
5591
5592         /*
5593          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
5594          * or else the reset will hang because there is no PCH to respond.
5595          * Move the handshake programming to initialization sequence.
5596          * Previously was left up to BIOS.
5597          */
5598         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5599         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
5600         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
5601
5602         /* Enable PG1 for cdclk */
5603         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5604
5605         /* check if cd clock is enabled */
5606         if (I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_PLL_ENABLE) {
5607                 DRM_DEBUG_KMS("Display already initialized\n");
5608                 return;
5609         }
5610
5611         /*
5612          * FIXME:
5613          * - The initial CDCLK needs to be read from VBT.
5614          *   Need to make this change after VBT has changes for BXT.
5615          * - check if setting the max (or any) cdclk freq is really necessary
5616          *   here, it belongs to modeset time
5617          */
5618         broxton_set_cdclk(dev, 624000);
5619
5620         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5621         POSTING_READ(DBUF_CTL);
5622
5623         udelay(10);
5624
5625         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5626                 DRM_ERROR("DBuf power enable timeout!\n");
5627 }
5628
5629 void broxton_uninit_cdclk(struct drm_device *dev)
5630 {
5631         struct drm_i915_private *dev_priv = dev->dev_private;
5632
5633         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5634         POSTING_READ(DBUF_CTL);
5635
5636         udelay(10);
5637
5638         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5639                 DRM_ERROR("DBuf power disable timeout!\n");
5640
5641         /* Set minimum (bypass) frequency, in effect turning off the DE PLL */
5642         broxton_set_cdclk(dev, 19200);
5643
5644         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5645 }
5646
5647 static const struct skl_cdclk_entry {
5648         unsigned int freq;
5649         unsigned int vco;
5650 } skl_cdclk_frequencies[] = {
5651         { .freq = 308570, .vco = 8640 },
5652         { .freq = 337500, .vco = 8100 },
5653         { .freq = 432000, .vco = 8640 },
5654         { .freq = 450000, .vco = 8100 },
5655         { .freq = 540000, .vco = 8100 },
5656         { .freq = 617140, .vco = 8640 },
5657         { .freq = 675000, .vco = 8100 },
5658 };
5659
5660 static unsigned int skl_cdclk_decimal(unsigned int freq)
5661 {
5662         return (freq - 1000) / 500;
5663 }
5664
5665 static unsigned int skl_cdclk_get_vco(unsigned int freq)
5666 {
5667         unsigned int i;
5668
5669         for (i = 0; i < ARRAY_SIZE(skl_cdclk_frequencies); i++) {
5670                 const struct skl_cdclk_entry *e = &skl_cdclk_frequencies[i];
5671
5672                 if (e->freq == freq)
5673                         return e->vco;
5674         }
5675
5676         return 8100;
5677 }
5678
5679 static void
5680 skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
5681 {
5682         unsigned int min_freq;
5683         u32 val;
5684
5685         /* select the minimum CDCLK before enabling DPLL 0 */
5686         val = I915_READ(CDCLK_CTL);
5687         val &= ~CDCLK_FREQ_SEL_MASK | ~CDCLK_FREQ_DECIMAL_MASK;
5688         val |= CDCLK_FREQ_337_308;
5689
5690         if (required_vco == 8640)
5691                 min_freq = 308570;
5692         else
5693                 min_freq = 337500;
5694
5695         val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_freq);
5696
5697         I915_WRITE(CDCLK_CTL, val);
5698         POSTING_READ(CDCLK_CTL);
5699
5700         /*
5701          * We always enable DPLL0 with the lowest link rate possible, but still
5702          * taking into account the VCO required to operate the eDP panel at the
5703          * desired frequency. The usual DP link rates operate with a VCO of
5704          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
5705          * The modeset code is responsible for the selection of the exact link
5706          * rate later on, with the constraint of choosing a frequency that
5707          * works with required_vco.
5708          */
5709         val = I915_READ(DPLL_CTRL1);
5710
5711         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
5712                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
5713         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
5714         if (required_vco == 8640)
5715                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
5716                                             SKL_DPLL0);
5717         else
5718                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810,
5719                                             SKL_DPLL0);
5720
5721         I915_WRITE(DPLL_CTRL1, val);
5722         POSTING_READ(DPLL_CTRL1);
5723
5724         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
5725
5726         if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5))
5727                 DRM_ERROR("DPLL0 not locked\n");
5728 }
5729
5730 static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
5731 {
5732         int ret;
5733         u32 val;
5734
5735         /* inform PCU we want to change CDCLK */
5736         val = SKL_CDCLK_PREPARE_FOR_CHANGE;
5737         mutex_lock(&dev_priv->rps.hw_lock);
5738         ret = sandybridge_pcode_read(dev_priv, SKL_PCODE_CDCLK_CONTROL, &val);
5739         mutex_unlock(&dev_priv->rps.hw_lock);
5740
5741         return ret == 0 && (val & SKL_CDCLK_READY_FOR_CHANGE);
5742 }
5743
5744 static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
5745 {
5746         unsigned int i;
5747
5748         for (i = 0; i < 15; i++) {
5749                 if (skl_cdclk_pcu_ready(dev_priv))
5750                         return true;
5751                 udelay(10);
5752         }
5753
5754         return false;
5755 }
5756
5757 static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
5758 {
5759         struct drm_device *dev = dev_priv->dev;
5760         u32 freq_select, pcu_ack;
5761
5762         DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq);
5763
5764         if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) {
5765                 DRM_ERROR("failed to inform PCU about cdclk change\n");
5766                 return;
5767         }
5768
5769         /* set CDCLK_CTL */
5770         switch(freq) {
5771         case 450000:
5772         case 432000:
5773                 freq_select = CDCLK_FREQ_450_432;
5774                 pcu_ack = 1;
5775                 break;
5776         case 540000:
5777                 freq_select = CDCLK_FREQ_540;
5778                 pcu_ack = 2;
5779                 break;
5780         case 308570:
5781         case 337500:
5782         default:
5783                 freq_select = CDCLK_FREQ_337_308;
5784                 pcu_ack = 0;
5785                 break;
5786         case 617140:
5787         case 675000:
5788                 freq_select = CDCLK_FREQ_675_617;
5789                 pcu_ack = 3;
5790                 break;
5791         }
5792
5793         I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(freq));
5794         POSTING_READ(CDCLK_CTL);
5795
5796         /* inform PCU of the change */
5797         mutex_lock(&dev_priv->rps.hw_lock);
5798         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
5799         mutex_unlock(&dev_priv->rps.hw_lock);
5800
5801         intel_update_cdclk(dev);
5802 }
5803
5804 void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
5805 {
5806         /* disable DBUF power */
5807         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5808         POSTING_READ(DBUF_CTL);
5809
5810         udelay(10);
5811
5812         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5813                 DRM_ERROR("DBuf power disable timeout\n");
5814
5815         /*
5816          * DMC assumes ownership of LCPLL and will get confused if we touch it.
5817          */
5818         if (dev_priv->csr.dmc_payload) {
5819                 /* disable DPLL0 */
5820                 I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) &
5821                                         ~LCPLL_PLL_ENABLE);
5822                 if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
5823                         DRM_ERROR("Couldn't disable DPLL0\n");
5824         }
5825
5826         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5827 }
5828
5829 void skl_init_cdclk(struct drm_i915_private *dev_priv)
5830 {
5831         u32 val;
5832         unsigned int required_vco;
5833
5834         /* enable PCH reset handshake */
5835         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5836         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
5837
5838         /* enable PG1 and Misc I/O */
5839         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5840
5841         /* DPLL0 not enabled (happens on early BIOS versions) */
5842         if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) {
5843                 /* enable DPLL0 */
5844                 required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk);
5845                 skl_dpll0_enable(dev_priv, required_vco);
5846         }
5847
5848         /* set CDCLK to the frequency the BIOS chose */
5849         skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk);
5850
5851         /* enable DBUF power */
5852         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5853         POSTING_READ(DBUF_CTL);
5854
5855         udelay(10);
5856
5857         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5858                 DRM_ERROR("DBuf power enable timeout\n");
5859 }
5860
5861 /* Adjust CDclk dividers to allow high res or save power if possible */
5862 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
5863 {
5864         struct drm_i915_private *dev_priv = dev->dev_private;
5865         u32 val, cmd;
5866
5867         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5868                                         != dev_priv->cdclk_freq);
5869
5870         if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
5871                 cmd = 2;
5872         else if (cdclk == 266667)
5873                 cmd = 1;
5874         else
5875                 cmd = 0;
5876
5877         mutex_lock(&dev_priv->rps.hw_lock);
5878         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5879         val &= ~DSPFREQGUAR_MASK;
5880         val |= (cmd << DSPFREQGUAR_SHIFT);
5881         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5882         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5883                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
5884                      50)) {
5885                 DRM_ERROR("timed out waiting for CDclk change\n");
5886         }
5887         mutex_unlock(&dev_priv->rps.hw_lock);
5888
5889         mutex_lock(&dev_priv->sb_lock);
5890
5891         if (cdclk == 400000) {
5892                 u32 divider;
5893
5894                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5895
5896                 /* adjust cdclk divider */
5897                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
5898                 val &= ~CCK_FREQUENCY_VALUES;
5899                 val |= divider;
5900                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
5901
5902                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
5903                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
5904                              50))
5905                         DRM_ERROR("timed out waiting for CDclk change\n");
5906         }
5907
5908         /* adjust self-refresh exit latency value */
5909         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
5910         val &= ~0x7f;
5911
5912         /*
5913          * For high bandwidth configs, we set a higher latency in the bunit
5914          * so that the core display fetch happens in time to avoid underruns.
5915          */
5916         if (cdclk == 400000)
5917                 val |= 4500 / 250; /* 4.5 usec */
5918         else
5919                 val |= 3000 / 250; /* 3.0 usec */
5920         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
5921
5922         mutex_unlock(&dev_priv->sb_lock);
5923
5924         intel_update_cdclk(dev);
5925 }
5926
5927 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
5928 {
5929         struct drm_i915_private *dev_priv = dev->dev_private;
5930         u32 val, cmd;
5931
5932         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5933                                                 != dev_priv->cdclk_freq);
5934
5935         switch (cdclk) {
5936         case 333333:
5937         case 320000:
5938         case 266667:
5939         case 200000:
5940                 break;
5941         default:
5942                 MISSING_CASE(cdclk);
5943                 return;
5944         }
5945
5946         /*
5947          * Specs are full of misinformation, but testing on actual
5948          * hardware has shown that we just need to write the desired
5949          * CCK divider into the Punit register.
5950          */
5951         cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5952
5953         mutex_lock(&dev_priv->rps.hw_lock);
5954         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5955         val &= ~DSPFREQGUAR_MASK_CHV;
5956         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
5957         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5958         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5959                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
5960                      50)) {
5961                 DRM_ERROR("timed out waiting for CDclk change\n");
5962         }
5963         mutex_unlock(&dev_priv->rps.hw_lock);
5964
5965         intel_update_cdclk(dev);
5966 }
5967
5968 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
5969                                  int max_pixclk)
5970 {
5971         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ? 333333 : 320000;
5972         int limit = IS_CHERRYVIEW(dev_priv) ? 95 : 90;
5973
5974         /*
5975          * Really only a few cases to deal with, as only 4 CDclks are supported:
5976          *   200MHz
5977          *   267MHz
5978          *   320/333MHz (depends on HPLL freq)
5979          *   400MHz (VLV only)
5980          * So we check to see whether we're above 90% (VLV) or 95% (CHV)
5981          * of the lower bin and adjust if needed.
5982          *
5983          * We seem to get an unstable or solid color picture at 200MHz.
5984          * Not sure what's wrong. For now use 200MHz only when all pipes
5985          * are off.
5986          */
5987         if (!IS_CHERRYVIEW(dev_priv) &&
5988             max_pixclk > freq_320*limit/100)
5989                 return 400000;
5990         else if (max_pixclk > 266667*limit/100)
5991                 return freq_320;
5992         else if (max_pixclk > 0)
5993                 return 266667;
5994         else
5995                 return 200000;
5996 }
5997
5998 static int broxton_calc_cdclk(struct drm_i915_private *dev_priv,
5999                               int max_pixclk)
6000 {
6001         /*
6002          * FIXME:
6003          * - remove the guardband, it's not needed on BXT
6004          * - set 19.2MHz bypass frequency if there are no active pipes
6005          */
6006         if (max_pixclk > 576000*9/10)
6007                 return 624000;
6008         else if (max_pixclk > 384000*9/10)
6009                 return 576000;
6010         else if (max_pixclk > 288000*9/10)
6011                 return 384000;
6012         else if (max_pixclk > 144000*9/10)
6013                 return 288000;
6014         else
6015                 return 144000;
6016 }
6017
6018 /* Compute the max pixel clock for new configuration. Uses atomic state if
6019  * that's non-NULL, look at current state otherwise. */
6020 static int intel_mode_max_pixclk(struct drm_device *dev,
6021                                  struct drm_atomic_state *state)
6022 {
6023         struct intel_crtc *intel_crtc;
6024         struct intel_crtc_state *crtc_state;
6025         int max_pixclk = 0;
6026
6027         for_each_intel_crtc(dev, intel_crtc) {
6028                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6029                 if (IS_ERR(crtc_state))
6030                         return PTR_ERR(crtc_state);
6031
6032                 if (!crtc_state->base.enable)
6033                         continue;
6034
6035                 max_pixclk = max(max_pixclk,
6036                                  crtc_state->base.adjusted_mode.crtc_clock);
6037         }
6038
6039         return max_pixclk;
6040 }
6041
6042 static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
6043 {
6044         struct drm_device *dev = state->dev;
6045         struct drm_i915_private *dev_priv = dev->dev_private;
6046         int max_pixclk = intel_mode_max_pixclk(dev, state);
6047
6048         if (max_pixclk < 0)
6049                 return max_pixclk;
6050
6051         to_intel_atomic_state(state)->cdclk =
6052                 valleyview_calc_cdclk(dev_priv, max_pixclk);
6053
6054         return 0;
6055 }
6056
6057 static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state)
6058 {
6059         struct drm_device *dev = state->dev;
6060         struct drm_i915_private *dev_priv = dev->dev_private;
6061         int max_pixclk = intel_mode_max_pixclk(dev, state);
6062
6063         if (max_pixclk < 0)
6064                 return max_pixclk;
6065
6066         to_intel_atomic_state(state)->cdclk =
6067                 broxton_calc_cdclk(dev_priv, max_pixclk);
6068
6069         return 0;
6070 }
6071
6072 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
6073 {
6074         unsigned int credits, default_credits;
6075
6076         if (IS_CHERRYVIEW(dev_priv))
6077                 default_credits = PFI_CREDIT(12);
6078         else
6079                 default_credits = PFI_CREDIT(8);
6080
6081         if (dev_priv->cdclk_freq >= dev_priv->czclk_freq) {
6082                 /* CHV suggested value is 31 or 63 */
6083                 if (IS_CHERRYVIEW(dev_priv))
6084                         credits = PFI_CREDIT_63;
6085                 else
6086                         credits = PFI_CREDIT(15);
6087         } else {
6088                 credits = default_credits;
6089         }
6090
6091         /*
6092          * WA - write default credits before re-programming
6093          * FIXME: should we also set the resend bit here?
6094          */
6095         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6096                    default_credits);
6097
6098         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6099                    credits | PFI_CREDIT_RESEND);
6100
6101         /*
6102          * FIXME is this guaranteed to clear
6103          * immediately or should we poll for it?
6104          */
6105         WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND);
6106 }
6107
6108 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
6109 {
6110         struct drm_device *dev = old_state->dev;
6111         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
6112         struct drm_i915_private *dev_priv = dev->dev_private;
6113
6114         /*
6115          * FIXME: We can end up here with all power domains off, yet
6116          * with a CDCLK frequency other than the minimum. To account
6117          * for this take the PIPE-A power domain, which covers the HW
6118          * blocks needed for the following programming. This can be
6119          * removed once it's guaranteed that we get here either with
6120          * the minimum CDCLK set, or the required power domains
6121          * enabled.
6122          */
6123         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
6124
6125         if (IS_CHERRYVIEW(dev))
6126                 cherryview_set_cdclk(dev, req_cdclk);
6127         else
6128                 valleyview_set_cdclk(dev, req_cdclk);
6129
6130         vlv_program_pfi_credits(dev_priv);
6131
6132         intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A);
6133 }
6134
6135 static void valleyview_crtc_enable(struct drm_crtc *crtc)
6136 {
6137         struct drm_device *dev = crtc->dev;
6138         struct drm_i915_private *dev_priv = to_i915(dev);
6139         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6140         struct intel_encoder *encoder;
6141         int pipe = intel_crtc->pipe;
6142         bool is_dsi;
6143
6144         if (WARN_ON(intel_crtc->active))
6145                 return;
6146
6147         is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
6148
6149         if (intel_crtc->config->has_dp_encoder)
6150                 intel_dp_set_m_n(intel_crtc, M1_N1);
6151
6152         intel_set_pipe_timings(intel_crtc);
6153
6154         if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
6155                 struct drm_i915_private *dev_priv = dev->dev_private;
6156
6157                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6158                 I915_WRITE(CHV_CANVAS(pipe), 0);
6159         }
6160
6161         i9xx_set_pipeconf(intel_crtc);
6162
6163         intel_crtc->active = true;
6164
6165         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6166
6167         for_each_encoder_on_crtc(dev, crtc, encoder)
6168                 if (encoder->pre_pll_enable)
6169                         encoder->pre_pll_enable(encoder);
6170
6171         if (!is_dsi) {
6172                 if (IS_CHERRYVIEW(dev)) {
6173                         chv_prepare_pll(intel_crtc, intel_crtc->config);
6174                         chv_enable_pll(intel_crtc, intel_crtc->config);
6175                 } else {
6176                         vlv_prepare_pll(intel_crtc, intel_crtc->config);
6177                         vlv_enable_pll(intel_crtc, intel_crtc->config);
6178                 }
6179         }
6180
6181         for_each_encoder_on_crtc(dev, crtc, encoder)
6182                 if (encoder->pre_enable)
6183                         encoder->pre_enable(encoder);
6184
6185         i9xx_pfit_enable(intel_crtc);
6186
6187         intel_crtc_load_lut(crtc);
6188
6189         intel_enable_pipe(intel_crtc);
6190
6191         assert_vblank_disabled(crtc);
6192         drm_crtc_vblank_on(crtc);
6193
6194         for_each_encoder_on_crtc(dev, crtc, encoder)
6195                 encoder->enable(encoder);
6196 }
6197
6198 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6199 {
6200         struct drm_device *dev = crtc->base.dev;
6201         struct drm_i915_private *dev_priv = dev->dev_private;
6202
6203         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6204         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6205 }
6206
6207 static void i9xx_crtc_enable(struct drm_crtc *crtc)
6208 {
6209         struct drm_device *dev = crtc->dev;
6210         struct drm_i915_private *dev_priv = to_i915(dev);
6211         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6212         struct intel_encoder *encoder;
6213         int pipe = intel_crtc->pipe;
6214
6215         if (WARN_ON(intel_crtc->active))
6216                 return;
6217
6218         i9xx_set_pll_dividers(intel_crtc);
6219
6220         if (intel_crtc->config->has_dp_encoder)
6221                 intel_dp_set_m_n(intel_crtc, M1_N1);
6222
6223         intel_set_pipe_timings(intel_crtc);
6224
6225         i9xx_set_pipeconf(intel_crtc);
6226
6227         intel_crtc->active = true;
6228
6229         if (!IS_GEN2(dev))
6230                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6231
6232         for_each_encoder_on_crtc(dev, crtc, encoder)
6233                 if (encoder->pre_enable)
6234                         encoder->pre_enable(encoder);
6235
6236         i9xx_enable_pll(intel_crtc);
6237
6238         i9xx_pfit_enable(intel_crtc);
6239
6240         intel_crtc_load_lut(crtc);
6241
6242         intel_update_watermarks(crtc);
6243         intel_enable_pipe(intel_crtc);
6244
6245         assert_vblank_disabled(crtc);
6246         drm_crtc_vblank_on(crtc);
6247
6248         for_each_encoder_on_crtc(dev, crtc, encoder)
6249                 encoder->enable(encoder);
6250 }
6251
6252 static void i9xx_pfit_disable(struct intel_crtc *crtc)
6253 {
6254         struct drm_device *dev = crtc->base.dev;
6255         struct drm_i915_private *dev_priv = dev->dev_private;
6256
6257         if (!crtc->config->gmch_pfit.control)
6258                 return;
6259
6260         assert_pipe_disabled(dev_priv, crtc->pipe);
6261
6262         DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n",
6263                          I915_READ(PFIT_CONTROL));
6264         I915_WRITE(PFIT_CONTROL, 0);
6265 }
6266
6267 static void i9xx_crtc_disable(struct drm_crtc *crtc)
6268 {
6269         struct drm_device *dev = crtc->dev;
6270         struct drm_i915_private *dev_priv = dev->dev_private;
6271         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6272         struct intel_encoder *encoder;
6273         int pipe = intel_crtc->pipe;
6274
6275         /*
6276          * On gen2 planes are double buffered but the pipe isn't, so we must
6277          * wait for planes to fully turn off before disabling the pipe.
6278          * We also need to wait on all gmch platforms because of the
6279          * self-refresh mode constraint explained above.
6280          */
6281         intel_wait_for_vblank(dev, pipe);
6282
6283         for_each_encoder_on_crtc(dev, crtc, encoder)
6284                 encoder->disable(encoder);
6285
6286         drm_crtc_vblank_off(crtc);
6287         assert_vblank_disabled(crtc);
6288
6289         intel_disable_pipe(intel_crtc);
6290
6291         i9xx_pfit_disable(intel_crtc);
6292
6293         for_each_encoder_on_crtc(dev, crtc, encoder)
6294                 if (encoder->post_disable)
6295                         encoder->post_disable(encoder);
6296
6297         if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) {
6298                 if (IS_CHERRYVIEW(dev))
6299                         chv_disable_pll(dev_priv, pipe);
6300                 else if (IS_VALLEYVIEW(dev))
6301                         vlv_disable_pll(dev_priv, pipe);
6302                 else
6303                         i9xx_disable_pll(intel_crtc);
6304         }
6305
6306         for_each_encoder_on_crtc(dev, crtc, encoder)
6307                 if (encoder->post_pll_disable)
6308                         encoder->post_pll_disable(encoder);
6309
6310         if (!IS_GEN2(dev))
6311                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6312 }
6313
6314 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
6315 {
6316         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6317         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6318         enum intel_display_power_domain domain;
6319         unsigned long domains;
6320
6321         if (!intel_crtc->active)
6322                 return;
6323
6324         if (to_intel_plane_state(crtc->primary->state)->visible) {
6325                 intel_crtc_wait_for_pending_flips(crtc);
6326                 intel_pre_disable_primary(crtc);
6327
6328                 intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
6329                 to_intel_plane_state(crtc->primary->state)->visible = false;
6330         }
6331
6332         dev_priv->display.crtc_disable(crtc);
6333         intel_crtc->active = false;
6334         intel_update_watermarks(crtc);
6335         intel_disable_shared_dpll(intel_crtc);
6336
6337         domains = intel_crtc->enabled_power_domains;
6338         for_each_power_domain(domain, domains)
6339                 intel_display_power_put(dev_priv, domain);
6340         intel_crtc->enabled_power_domains = 0;
6341 }
6342
6343 /*
6344  * turn all crtc's off, but do not adjust state
6345  * This has to be paired with a call to intel_modeset_setup_hw_state.
6346  */
6347 int intel_display_suspend(struct drm_device *dev)
6348 {
6349         struct drm_mode_config *config = &dev->mode_config;
6350         struct drm_modeset_acquire_ctx *ctx = config->acquire_ctx;
6351         struct drm_atomic_state *state;
6352         struct drm_crtc *crtc;
6353         unsigned crtc_mask = 0;
6354         int ret = 0;
6355
6356         if (WARN_ON(!ctx))
6357                 return 0;
6358
6359         lockdep_assert_held(&ctx->ww_ctx);
6360         state = drm_atomic_state_alloc(dev);
6361         if (WARN_ON(!state))
6362                 return -ENOMEM;
6363
6364         state->acquire_ctx = ctx;
6365         state->allow_modeset = true;
6366
6367         for_each_crtc(dev, crtc) {
6368                 struct drm_crtc_state *crtc_state =
6369                         drm_atomic_get_crtc_state(state, crtc);
6370
6371                 ret = PTR_ERR_OR_ZERO(crtc_state);
6372                 if (ret)
6373                         goto free;
6374
6375                 if (!crtc_state->active)
6376                         continue;
6377
6378                 crtc_state->active = false;
6379                 crtc_mask |= 1 << drm_crtc_index(crtc);
6380         }
6381
6382         if (crtc_mask) {
6383                 ret = drm_atomic_commit(state);
6384
6385                 if (!ret) {
6386                         for_each_crtc(dev, crtc)
6387                                 if (crtc_mask & (1 << drm_crtc_index(crtc)))
6388                                         crtc->state->active = true;
6389
6390                         return ret;
6391                 }
6392         }
6393
6394 free:
6395         if (ret)
6396                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6397         drm_atomic_state_free(state);
6398         return ret;
6399 }
6400
6401 void intel_encoder_destroy(struct drm_encoder *encoder)
6402 {
6403         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6404
6405         drm_encoder_cleanup(encoder);
6406         kfree(intel_encoder);
6407 }
6408
6409 /* Cross check the actual hw state with our own modeset state tracking (and it's
6410  * internal consistency). */
6411 static void intel_connector_check_state(struct intel_connector *connector)
6412 {
6413         struct drm_crtc *crtc = connector->base.state->crtc;
6414
6415         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6416                       connector->base.base.id,
6417                       connector->base.name);
6418
6419         if (connector->get_hw_state(connector)) {
6420                 struct intel_encoder *encoder = connector->encoder;
6421                 struct drm_connector_state *conn_state = connector->base.state;
6422
6423                 I915_STATE_WARN(!crtc,
6424                          "connector enabled without attached crtc\n");
6425
6426                 if (!crtc)
6427                         return;
6428
6429                 I915_STATE_WARN(!crtc->state->active,
6430                       "connector is active, but attached crtc isn't\n");
6431
6432                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6433                         return;
6434
6435                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6436                         "atomic encoder doesn't match attached encoder\n");
6437
6438                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6439                         "attached encoder crtc differs from connector crtc\n");
6440         } else {
6441                 I915_STATE_WARN(crtc && crtc->state->active,
6442                         "attached crtc is active, but connector isn't\n");
6443                 I915_STATE_WARN(!crtc && connector->base.state->best_encoder,
6444                         "best encoder set without crtc!\n");
6445         }
6446 }
6447
6448 int intel_connector_init(struct intel_connector *connector)
6449 {
6450         struct drm_connector_state *connector_state;
6451
6452         connector_state = kzalloc(sizeof *connector_state, GFP_KERNEL);
6453         if (!connector_state)
6454                 return -ENOMEM;
6455
6456         connector->base.state = connector_state;
6457         return 0;
6458 }
6459
6460 struct intel_connector *intel_connector_alloc(void)
6461 {
6462         struct intel_connector *connector;
6463
6464         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6465         if (!connector)
6466                 return NULL;
6467
6468         if (intel_connector_init(connector) < 0) {
6469                 kfree(connector);
6470                 return NULL;
6471         }
6472
6473         return connector;
6474 }
6475
6476 /* Simple connector->get_hw_state implementation for encoders that support only
6477  * one connector and no cloning and hence the encoder state determines the state
6478  * of the connector. */
6479 bool intel_connector_get_hw_state(struct intel_connector *connector)
6480 {
6481         enum pipe pipe = 0;
6482         struct intel_encoder *encoder = connector->encoder;
6483
6484         return encoder->get_hw_state(encoder, &pipe);
6485 }
6486
6487 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6488 {
6489         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6490                 return crtc_state->fdi_lanes;
6491
6492         return 0;
6493 }
6494
6495 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6496                                      struct intel_crtc_state *pipe_config)
6497 {
6498         struct drm_atomic_state *state = pipe_config->base.state;
6499         struct intel_crtc *other_crtc;
6500         struct intel_crtc_state *other_crtc_state;
6501
6502         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6503                       pipe_name(pipe), pipe_config->fdi_lanes);
6504         if (pipe_config->fdi_lanes > 4) {
6505                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6506                               pipe_name(pipe), pipe_config->fdi_lanes);
6507                 return -EINVAL;
6508         }
6509
6510         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
6511                 if (pipe_config->fdi_lanes > 2) {
6512                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6513                                       pipe_config->fdi_lanes);
6514                         return -EINVAL;
6515                 } else {
6516                         return 0;
6517                 }
6518         }
6519
6520         if (INTEL_INFO(dev)->num_pipes == 2)
6521                 return 0;
6522
6523         /* Ivybridge 3 pipe is really complicated */
6524         switch (pipe) {
6525         case PIPE_A:
6526                 return 0;
6527         case PIPE_B:
6528                 if (pipe_config->fdi_lanes <= 2)
6529                         return 0;
6530
6531                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C));
6532                 other_crtc_state =
6533                         intel_atomic_get_crtc_state(state, other_crtc);
6534                 if (IS_ERR(other_crtc_state))
6535                         return PTR_ERR(other_crtc_state);
6536
6537                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6538                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6539                                       pipe_name(pipe), pipe_config->fdi_lanes);
6540                         return -EINVAL;
6541                 }
6542                 return 0;
6543         case PIPE_C:
6544                 if (pipe_config->fdi_lanes > 2) {
6545                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6546                                       pipe_name(pipe), pipe_config->fdi_lanes);
6547                         return -EINVAL;
6548                 }
6549
6550                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B));
6551                 other_crtc_state =
6552                         intel_atomic_get_crtc_state(state, other_crtc);
6553                 if (IS_ERR(other_crtc_state))
6554                         return PTR_ERR(other_crtc_state);
6555
6556                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6557                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6558                         return -EINVAL;
6559                 }
6560                 return 0;
6561         default:
6562                 BUG();
6563         }
6564 }
6565
6566 #define RETRY 1
6567 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6568                                        struct intel_crtc_state *pipe_config)
6569 {
6570         struct drm_device *dev = intel_crtc->base.dev;
6571         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6572         int lane, link_bw, fdi_dotclock, ret;
6573         bool needs_recompute = false;
6574
6575 retry:
6576         /* FDI is a binary signal running at ~2.7GHz, encoding
6577          * each output octet as 10 bits. The actual frequency
6578          * is stored as a divider into a 100MHz clock, and the
6579          * mode pixel clock is stored in units of 1KHz.
6580          * Hence the bw of each lane in terms of the mode signal
6581          * is:
6582          */
6583         link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
6584
6585         fdi_dotclock = adjusted_mode->crtc_clock;
6586
6587         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6588                                            pipe_config->pipe_bpp);
6589
6590         pipe_config->fdi_lanes = lane;
6591
6592         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6593                                link_bw, &pipe_config->fdi_m_n);
6594
6595         ret = ironlake_check_fdi_lanes(intel_crtc->base.dev,
6596                                        intel_crtc->pipe, pipe_config);
6597         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6598                 pipe_config->pipe_bpp -= 2*3;
6599                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6600                               pipe_config->pipe_bpp);
6601                 needs_recompute = true;
6602                 pipe_config->bw_constrained = true;
6603
6604                 goto retry;
6605         }
6606
6607         if (needs_recompute)
6608                 return RETRY;
6609
6610         return ret;
6611 }
6612
6613 static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
6614                                      struct intel_crtc_state *pipe_config)
6615 {
6616         if (pipe_config->pipe_bpp > 24)
6617                 return false;
6618
6619         /* HSW can handle pixel rate up to cdclk? */
6620         if (IS_HASWELL(dev_priv->dev))
6621                 return true;
6622
6623         /*
6624          * We compare against max which means we must take
6625          * the increased cdclk requirement into account when
6626          * calculating the new cdclk.
6627          *
6628          * Should measure whether using a lower cdclk w/o IPS
6629          */
6630         return ilk_pipe_pixel_rate(pipe_config) <=
6631                 dev_priv->max_cdclk_freq * 95 / 100;
6632 }
6633
6634 static void hsw_compute_ips_config(struct intel_crtc *crtc,
6635                                    struct intel_crtc_state *pipe_config)
6636 {
6637         struct drm_device *dev = crtc->base.dev;
6638         struct drm_i915_private *dev_priv = dev->dev_private;
6639
6640         pipe_config->ips_enabled = i915.enable_ips &&
6641                 hsw_crtc_supports_ips(crtc) &&
6642                 pipe_config_supports_ips(dev_priv, pipe_config);
6643 }
6644
6645 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6646                                      struct intel_crtc_state *pipe_config)
6647 {
6648         struct drm_device *dev = crtc->base.dev;
6649         struct drm_i915_private *dev_priv = dev->dev_private;
6650         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6651
6652         /* FIXME should check pixel clock limits on all platforms */
6653         if (INTEL_INFO(dev)->gen < 4) {
6654                 int clock_limit = dev_priv->max_cdclk_freq;
6655
6656                 /*
6657                  * Enable pixel doubling when the dot clock
6658                  * is > 90% of the (display) core speed.
6659                  *
6660                  * GDG double wide on either pipe,
6661                  * otherwise pipe A only.
6662                  */
6663                 if ((crtc->pipe == PIPE_A || IS_I915G(dev)) &&
6664                     adjusted_mode->crtc_clock > clock_limit * 9 / 10) {
6665                         clock_limit *= 2;
6666                         pipe_config->double_wide = true;
6667                 }
6668
6669                 if (adjusted_mode->crtc_clock > clock_limit * 9 / 10)
6670                         return -EINVAL;
6671         }
6672
6673         /*
6674          * Pipe horizontal size must be even in:
6675          * - DVO ganged mode
6676          * - LVDS dual channel mode
6677          * - Double wide pipe
6678          */
6679         if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6680              intel_is_dual_link_lvds(dev)) || pipe_config->double_wide)
6681                 pipe_config->pipe_src_w &= ~1;
6682
6683         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6684          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6685          */
6686         if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
6687                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6688                 return -EINVAL;
6689
6690         if (HAS_IPS(dev))
6691                 hsw_compute_ips_config(crtc, pipe_config);
6692
6693         if (pipe_config->has_pch_encoder)
6694                 return ironlake_fdi_compute_config(crtc, pipe_config);
6695
6696         return 0;
6697 }
6698
6699 static int skylake_get_display_clock_speed(struct drm_device *dev)
6700 {
6701         struct drm_i915_private *dev_priv = to_i915(dev);
6702         uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
6703         uint32_t cdctl = I915_READ(CDCLK_CTL);
6704         uint32_t linkrate;
6705
6706         if (!(lcpll1 & LCPLL_PLL_ENABLE))
6707                 return 24000; /* 24MHz is the cd freq with NSSC ref */
6708
6709         if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540)
6710                 return 540000;
6711
6712         linkrate = (I915_READ(DPLL_CTRL1) &
6713                     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) >> 1;
6714
6715         if (linkrate == DPLL_CTRL1_LINK_RATE_2160 ||
6716             linkrate == DPLL_CTRL1_LINK_RATE_1080) {
6717                 /* vco 8640 */
6718                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6719                 case CDCLK_FREQ_450_432:
6720                         return 432000;
6721                 case CDCLK_FREQ_337_308:
6722                         return 308570;
6723                 case CDCLK_FREQ_675_617:
6724                         return 617140;
6725                 default:
6726                         WARN(1, "Unknown cd freq selection\n");
6727                 }
6728         } else {
6729                 /* vco 8100 */
6730                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6731                 case CDCLK_FREQ_450_432:
6732                         return 450000;
6733                 case CDCLK_FREQ_337_308:
6734                         return 337500;
6735                 case CDCLK_FREQ_675_617:
6736                         return 675000;
6737                 default:
6738                         WARN(1, "Unknown cd freq selection\n");
6739                 }
6740         }
6741
6742         /* error case, do as if DPLL0 isn't enabled */
6743         return 24000;
6744 }
6745
6746 static int broxton_get_display_clock_speed(struct drm_device *dev)
6747 {
6748         struct drm_i915_private *dev_priv = to_i915(dev);
6749         uint32_t cdctl = I915_READ(CDCLK_CTL);
6750         uint32_t pll_ratio = I915_READ(BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
6751         uint32_t pll_enab = I915_READ(BXT_DE_PLL_ENABLE);
6752         int cdclk;
6753
6754         if (!(pll_enab & BXT_DE_PLL_PLL_ENABLE))
6755                 return 19200;
6756
6757         cdclk = 19200 * pll_ratio / 2;
6758
6759         switch (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) {
6760         case BXT_CDCLK_CD2X_DIV_SEL_1:
6761                 return cdclk;  /* 576MHz or 624MHz */
6762         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
6763                 return cdclk * 2 / 3; /* 384MHz */
6764         case BXT_CDCLK_CD2X_DIV_SEL_2:
6765                 return cdclk / 2; /* 288MHz */
6766         case BXT_CDCLK_CD2X_DIV_SEL_4:
6767                 return cdclk / 4; /* 144MHz */
6768         }
6769
6770         /* error case, do as if DE PLL isn't enabled */
6771         return 19200;
6772 }
6773
6774 static int broadwell_get_display_clock_speed(struct drm_device *dev)
6775 {
6776         struct drm_i915_private *dev_priv = dev->dev_private;
6777         uint32_t lcpll = I915_READ(LCPLL_CTL);
6778         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6779
6780         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6781                 return 800000;
6782         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6783                 return 450000;
6784         else if (freq == LCPLL_CLK_FREQ_450)
6785                 return 450000;
6786         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
6787                 return 540000;
6788         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
6789                 return 337500;
6790         else
6791                 return 675000;
6792 }
6793
6794 static int haswell_get_display_clock_speed(struct drm_device *dev)
6795 {
6796         struct drm_i915_private *dev_priv = dev->dev_private;
6797         uint32_t lcpll = I915_READ(LCPLL_CTL);
6798         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6799
6800         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6801                 return 800000;
6802         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6803                 return 450000;
6804         else if (freq == LCPLL_CLK_FREQ_450)
6805                 return 450000;
6806         else if (IS_HSW_ULT(dev))
6807                 return 337500;
6808         else
6809                 return 540000;
6810 }
6811
6812 static int valleyview_get_display_clock_speed(struct drm_device *dev)
6813 {
6814         return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk",
6815                                       CCK_DISPLAY_CLOCK_CONTROL);
6816 }
6817
6818 static int ilk_get_display_clock_speed(struct drm_device *dev)
6819 {
6820         return 450000;
6821 }
6822
6823 static int i945_get_display_clock_speed(struct drm_device *dev)
6824 {
6825         return 400000;
6826 }
6827
6828 static int i915_get_display_clock_speed(struct drm_device *dev)
6829 {
6830         return 333333;
6831 }
6832
6833 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
6834 {
6835         return 200000;
6836 }
6837
6838 static int pnv_get_display_clock_speed(struct drm_device *dev)
6839 {
6840         u16 gcfgc = 0;
6841
6842         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6843
6844         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6845         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
6846                 return 266667;
6847         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
6848                 return 333333;
6849         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
6850                 return 444444;
6851         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
6852                 return 200000;
6853         default:
6854                 DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
6855         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
6856                 return 133333;
6857         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
6858                 return 166667;
6859         }
6860 }
6861
6862 static int i915gm_get_display_clock_speed(struct drm_device *dev)
6863 {
6864         u16 gcfgc = 0;
6865
6866         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6867
6868         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
6869                 return 133333;
6870         else {
6871                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6872                 case GC_DISPLAY_CLOCK_333_MHZ:
6873                         return 333333;
6874                 default:
6875                 case GC_DISPLAY_CLOCK_190_200_MHZ:
6876                         return 190000;
6877                 }
6878         }
6879 }
6880
6881 static int i865_get_display_clock_speed(struct drm_device *dev)
6882 {
6883         return 266667;
6884 }
6885
6886 static int i85x_get_display_clock_speed(struct drm_device *dev)
6887 {
6888         u16 hpllcc = 0;
6889
6890         /*
6891          * 852GM/852GMV only supports 133 MHz and the HPLLCC
6892          * encoding is different :(
6893          * FIXME is this the right way to detect 852GM/852GMV?
6894          */
6895         if (dev->pdev->revision == 0x1)
6896                 return 133333;
6897
6898         pci_bus_read_config_word(dev->pdev->bus,
6899                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
6900
6901         /* Assume that the hardware is in the high speed state.  This
6902          * should be the default.
6903          */
6904         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
6905         case GC_CLOCK_133_200:
6906         case GC_CLOCK_133_200_2:
6907         case GC_CLOCK_100_200:
6908                 return 200000;
6909         case GC_CLOCK_166_250:
6910                 return 250000;
6911         case GC_CLOCK_100_133:
6912                 return 133333;
6913         case GC_CLOCK_133_266:
6914         case GC_CLOCK_133_266_2:
6915         case GC_CLOCK_166_266:
6916                 return 266667;
6917         }
6918
6919         /* Shouldn't happen */
6920         return 0;
6921 }
6922
6923 static int i830_get_display_clock_speed(struct drm_device *dev)
6924 {
6925         return 133333;
6926 }
6927
6928 static unsigned int intel_hpll_vco(struct drm_device *dev)
6929 {
6930         struct drm_i915_private *dev_priv = dev->dev_private;
6931         static const unsigned int blb_vco[8] = {
6932                 [0] = 3200000,
6933                 [1] = 4000000,
6934                 [2] = 5333333,
6935                 [3] = 4800000,
6936                 [4] = 6400000,
6937         };
6938         static const unsigned int pnv_vco[8] = {
6939                 [0] = 3200000,
6940                 [1] = 4000000,
6941                 [2] = 5333333,
6942                 [3] = 4800000,
6943                 [4] = 2666667,
6944         };
6945         static const unsigned int cl_vco[8] = {
6946                 [0] = 3200000,
6947                 [1] = 4000000,
6948                 [2] = 5333333,
6949                 [3] = 6400000,
6950                 [4] = 3333333,
6951                 [5] = 3566667,
6952                 [6] = 4266667,
6953         };
6954         static const unsigned int elk_vco[8] = {
6955                 [0] = 3200000,
6956                 [1] = 4000000,
6957                 [2] = 5333333,
6958                 [3] = 4800000,
6959         };
6960         static const unsigned int ctg_vco[8] = {
6961                 [0] = 3200000,
6962                 [1] = 4000000,
6963                 [2] = 5333333,
6964                 [3] = 6400000,
6965                 [4] = 2666667,
6966                 [5] = 4266667,
6967         };
6968         const unsigned int *vco_table;
6969         unsigned int vco;
6970         uint8_t tmp = 0;
6971
6972         /* FIXME other chipsets? */
6973         if (IS_GM45(dev))
6974                 vco_table = ctg_vco;
6975         else if (IS_G4X(dev))
6976                 vco_table = elk_vco;
6977         else if (IS_CRESTLINE(dev))
6978                 vco_table = cl_vco;
6979         else if (IS_PINEVIEW(dev))
6980                 vco_table = pnv_vco;
6981         else if (IS_G33(dev))
6982                 vco_table = blb_vco;
6983         else
6984                 return 0;
6985
6986         tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO);
6987
6988         vco = vco_table[tmp & 0x7];
6989         if (vco == 0)
6990                 DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp);
6991         else
6992                 DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco);
6993
6994         return vco;
6995 }
6996
6997 static int gm45_get_display_clock_speed(struct drm_device *dev)
6998 {
6999         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7000         uint16_t tmp = 0;
7001
7002         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7003
7004         cdclk_sel = (tmp >> 12) & 0x1;
7005
7006         switch (vco) {
7007         case 2666667:
7008         case 4000000:
7009         case 5333333:
7010                 return cdclk_sel ? 333333 : 222222;
7011         case 3200000:
7012                 return cdclk_sel ? 320000 : 228571;
7013         default:
7014                 DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", vco, tmp);
7015                 return 222222;
7016         }
7017 }
7018
7019 static int i965gm_get_display_clock_speed(struct drm_device *dev)
7020 {
7021         static const uint8_t div_3200[] = { 16, 10,  8 };
7022         static const uint8_t div_4000[] = { 20, 12, 10 };
7023         static const uint8_t div_5333[] = { 24, 16, 14 };
7024         const uint8_t *div_table;
7025         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7026         uint16_t tmp = 0;
7027
7028         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7029
7030         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
7031
7032         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7033                 goto fail;
7034
7035         switch (vco) {
7036         case 3200000:
7037                 div_table = div_3200;
7038                 break;
7039         case 4000000:
7040                 div_table = div_4000;
7041                 break;
7042         case 5333333:
7043                 div_table = div_5333;
7044                 break;
7045         default:
7046                 goto fail;
7047         }
7048
7049         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7050
7051 fail:
7052         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", vco, tmp);
7053         return 200000;
7054 }
7055
7056 static int g33_get_display_clock_speed(struct drm_device *dev)
7057 {
7058         static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
7059         static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
7060         static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
7061         static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 };
7062         const uint8_t *div_table;
7063         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7064         uint16_t tmp = 0;
7065
7066         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7067
7068         cdclk_sel = (tmp >> 4) & 0x7;
7069
7070         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7071                 goto fail;
7072
7073         switch (vco) {
7074         case 3200000:
7075                 div_table = div_3200;
7076                 break;
7077         case 4000000:
7078                 div_table = div_4000;
7079                 break;
7080         case 4800000:
7081                 div_table = div_4800;
7082                 break;
7083         case 5333333:
7084                 div_table = div_5333;
7085                 break;
7086         default:
7087                 goto fail;
7088         }
7089
7090         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7091
7092 fail:
7093         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", vco, tmp);
7094         return 190476;
7095 }
7096
7097 static void
7098 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
7099 {
7100         while (*num > DATA_LINK_M_N_MASK ||
7101                *den > DATA_LINK_M_N_MASK) {
7102                 *num >>= 1;
7103                 *den >>= 1;
7104         }
7105 }
7106
7107 static void compute_m_n(unsigned int m, unsigned int n,
7108                         uint32_t *ret_m, uint32_t *ret_n)
7109 {
7110         *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7111         *ret_m = div_u64((uint64_t) m * *ret_n, n);
7112         intel_reduce_m_n_ratio(ret_m, ret_n);
7113 }
7114
7115 void
7116 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
7117                        int pixel_clock, int link_clock,
7118                        struct intel_link_m_n *m_n)
7119 {
7120         m_n->tu = 64;
7121
7122         compute_m_n(bits_per_pixel * pixel_clock,
7123                     link_clock * nlanes * 8,
7124                     &m_n->gmch_m, &m_n->gmch_n);
7125
7126         compute_m_n(pixel_clock, link_clock,
7127                     &m_n->link_m, &m_n->link_n);
7128 }
7129
7130 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7131 {
7132         if (i915.panel_use_ssc >= 0)
7133                 return i915.panel_use_ssc != 0;
7134         return dev_priv->vbt.lvds_use_ssc
7135                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7136 }
7137
7138 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
7139                            int num_connectors)
7140 {
7141         struct drm_device *dev = crtc_state->base.crtc->dev;
7142         struct drm_i915_private *dev_priv = dev->dev_private;
7143         int refclk;
7144
7145         WARN_ON(!crtc_state->base.state);
7146
7147         if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) {
7148                 refclk = 100000;
7149         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7150             intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
7151                 refclk = dev_priv->vbt.lvds_ssc_freq;
7152                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7153         } else if (!IS_GEN2(dev)) {
7154                 refclk = 96000;
7155         } else {
7156                 refclk = 48000;
7157         }
7158
7159         return refclk;
7160 }
7161
7162 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
7163 {
7164         return (1 << dpll->n) << 16 | dpll->m2;
7165 }
7166
7167 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
7168 {
7169         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7170 }
7171
7172 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7173                                      struct intel_crtc_state *crtc_state,
7174                                      intel_clock_t *reduced_clock)
7175 {
7176         struct drm_device *dev = crtc->base.dev;
7177         u32 fp, fp2 = 0;
7178
7179         if (IS_PINEVIEW(dev)) {
7180                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7181                 if (reduced_clock)
7182                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7183         } else {
7184                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7185                 if (reduced_clock)
7186                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7187         }
7188
7189         crtc_state->dpll_hw_state.fp0 = fp;
7190
7191         crtc->lowfreq_avail = false;
7192         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7193             reduced_clock) {
7194                 crtc_state->dpll_hw_state.fp1 = fp2;
7195                 crtc->lowfreq_avail = true;
7196         } else {
7197                 crtc_state->dpll_hw_state.fp1 = fp;
7198         }
7199 }
7200
7201 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7202                 pipe)
7203 {
7204         u32 reg_val;
7205
7206         /*
7207          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7208          * and set it to a reasonable value instead.
7209          */
7210         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7211         reg_val &= 0xffffff00;
7212         reg_val |= 0x00000030;
7213         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7214
7215         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7216         reg_val &= 0x8cffffff;
7217         reg_val = 0x8c000000;
7218         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7219
7220         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7221         reg_val &= 0xffffff00;
7222         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7223
7224         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7225         reg_val &= 0x00ffffff;
7226         reg_val |= 0xb0000000;
7227         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7228 }
7229
7230 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
7231                                          struct intel_link_m_n *m_n)
7232 {
7233         struct drm_device *dev = crtc->base.dev;
7234         struct drm_i915_private *dev_priv = dev->dev_private;
7235         int pipe = crtc->pipe;
7236
7237         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7238         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7239         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7240         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7241 }
7242
7243 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
7244                                          struct intel_link_m_n *m_n,
7245                                          struct intel_link_m_n *m2_n2)
7246 {
7247         struct drm_device *dev = crtc->base.dev;
7248         struct drm_i915_private *dev_priv = dev->dev_private;
7249         int pipe = crtc->pipe;
7250         enum transcoder transcoder = crtc->config->cpu_transcoder;
7251
7252         if (INTEL_INFO(dev)->gen >= 5) {
7253                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7254                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7255                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7256                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7257                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
7258                  * for gen < 8) and if DRRS is supported (to make sure the
7259                  * registers are not unnecessarily accessed).
7260                  */
7261                 if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
7262                         crtc->config->has_drrs) {
7263                         I915_WRITE(PIPE_DATA_M2(transcoder),
7264                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7265                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7266                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7267                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7268                 }
7269         } else {
7270                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7271                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7272                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7273                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7274         }
7275 }
7276
7277 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
7278 {
7279         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7280
7281         if (m_n == M1_N1) {
7282                 dp_m_n = &crtc->config->dp_m_n;
7283                 dp_m2_n2 = &crtc->config->dp_m2_n2;
7284         } else if (m_n == M2_N2) {
7285
7286                 /*
7287                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7288                  * needs to be programmed into M1_N1.
7289                  */
7290                 dp_m_n = &crtc->config->dp_m2_n2;
7291         } else {
7292                 DRM_ERROR("Unsupported divider value\n");
7293                 return;
7294         }
7295
7296         if (crtc->config->has_pch_encoder)
7297                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
7298         else
7299                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
7300 }
7301
7302 static void vlv_compute_dpll(struct intel_crtc *crtc,
7303                              struct intel_crtc_state *pipe_config)
7304 {
7305         u32 dpll, dpll_md;
7306
7307         /*
7308          * Enable DPIO clock input. We should never disable the reference
7309          * clock for pipe B, since VGA hotplug / manual detection depends
7310          * on it.
7311          */
7312         dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REF_CLK_ENABLE_VLV |
7313                 DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_REF_CLK_VLV;
7314         /* We should never disable this, set it here for state tracking */
7315         if (crtc->pipe == PIPE_B)
7316                 dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7317         dpll |= DPLL_VCO_ENABLE;
7318         pipe_config->dpll_hw_state.dpll = dpll;
7319
7320         dpll_md = (pipe_config->pixel_multiplier - 1)
7321                 << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7322         pipe_config->dpll_hw_state.dpll_md = dpll_md;
7323 }
7324
7325 static void vlv_prepare_pll(struct intel_crtc *crtc,
7326                             const struct intel_crtc_state *pipe_config)
7327 {
7328         struct drm_device *dev = crtc->base.dev;
7329         struct drm_i915_private *dev_priv = dev->dev_private;
7330         int pipe = crtc->pipe;
7331         u32 mdiv;
7332         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7333         u32 coreclk, reg_val;
7334
7335         mutex_lock(&dev_priv->sb_lock);
7336
7337         bestn = pipe_config->dpll.n;
7338         bestm1 = pipe_config->dpll.m1;
7339         bestm2 = pipe_config->dpll.m2;
7340         bestp1 = pipe_config->dpll.p1;
7341         bestp2 = pipe_config->dpll.p2;
7342
7343         /* See eDP HDMI DPIO driver vbios notes doc */
7344
7345         /* PLL B needs special handling */
7346         if (pipe == PIPE_B)
7347                 vlv_pllb_recal_opamp(dev_priv, pipe);
7348
7349         /* Set up Tx target for periodic Rcomp update */
7350         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7351
7352         /* Disable target IRef on PLL */
7353         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7354         reg_val &= 0x00ffffff;
7355         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7356
7357         /* Disable fast lock */
7358         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7359
7360         /* Set idtafcrecal before PLL is enabled */
7361         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7362         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7363         mdiv |= ((bestn << DPIO_N_SHIFT));
7364         mdiv |= (1 << DPIO_K_SHIFT);
7365
7366         /*
7367          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7368          * but we don't support that).
7369          * Note: don't use the DAC post divider as it seems unstable.
7370          */
7371         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7372         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7373
7374         mdiv |= DPIO_ENABLE_CALIBRATION;
7375         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7376
7377         /* Set HBR and RBR LPF coefficients */
7378         if (pipe_config->port_clock == 162000 ||
7379             intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) ||
7380             intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
7381                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7382                                  0x009f0003);
7383         else
7384                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7385                                  0x00d0000f);
7386
7387         if (pipe_config->has_dp_encoder) {
7388                 /* Use SSC source */
7389                 if (pipe == PIPE_A)
7390                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7391                                          0x0df40000);
7392                 else
7393                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7394                                          0x0df70000);
7395         } else { /* HDMI or VGA */
7396                 /* Use bend source */
7397                 if (pipe == PIPE_A)
7398                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7399                                          0x0df70000);
7400                 else
7401                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7402                                          0x0df40000);
7403         }
7404
7405         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7406         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7407         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
7408             intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
7409                 coreclk |= 0x01000000;
7410         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7411
7412         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7413         mutex_unlock(&dev_priv->sb_lock);
7414 }
7415
7416 static void chv_compute_dpll(struct intel_crtc *crtc,
7417                              struct intel_crtc_state *pipe_config)
7418 {
7419         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7420                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
7421                 DPLL_VCO_ENABLE;
7422         if (crtc->pipe != PIPE_A)
7423                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7424
7425         pipe_config->dpll_hw_state.dpll_md =
7426                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7427 }
7428
7429 static void chv_prepare_pll(struct intel_crtc *crtc,
7430                             const struct intel_crtc_state *pipe_config)
7431 {
7432         struct drm_device *dev = crtc->base.dev;
7433         struct drm_i915_private *dev_priv = dev->dev_private;
7434         int pipe = crtc->pipe;
7435         int dpll_reg = DPLL(crtc->pipe);
7436         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7437         u32 loopfilter, tribuf_calcntr;
7438         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7439         u32 dpio_val;
7440         int vco;
7441
7442         bestn = pipe_config->dpll.n;
7443         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7444         bestm1 = pipe_config->dpll.m1;
7445         bestm2 = pipe_config->dpll.m2 >> 22;
7446         bestp1 = pipe_config->dpll.p1;
7447         bestp2 = pipe_config->dpll.p2;
7448         vco = pipe_config->dpll.vco;
7449         dpio_val = 0;
7450         loopfilter = 0;
7451
7452         /*
7453          * Enable Refclk and SSC
7454          */
7455         I915_WRITE(dpll_reg,
7456                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7457
7458         mutex_lock(&dev_priv->sb_lock);
7459
7460         /* p1 and p2 divider */
7461         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7462                         5 << DPIO_CHV_S1_DIV_SHIFT |
7463                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7464                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7465                         1 << DPIO_CHV_K_DIV_SHIFT);
7466
7467         /* Feedback post-divider - m2 */
7468         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7469
7470         /* Feedback refclk divider - n and m1 */
7471         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7472                         DPIO_CHV_M1_DIV_BY_2 |
7473                         1 << DPIO_CHV_N_DIV_SHIFT);
7474
7475         /* M2 fraction division */
7476         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7477
7478         /* M2 fraction division enable */
7479         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7480         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7481         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7482         if (bestm2_frac)
7483                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7484         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7485
7486         /* Program digital lock detect threshold */
7487         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7488         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7489                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7490         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7491         if (!bestm2_frac)
7492                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7493         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7494
7495         /* Loop filter */
7496         if (vco == 5400000) {
7497                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7498                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7499                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7500                 tribuf_calcntr = 0x9;
7501         } else if (vco <= 6200000) {
7502                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7503                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7504                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7505                 tribuf_calcntr = 0x9;
7506         } else if (vco <= 6480000) {
7507                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7508                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7509                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7510                 tribuf_calcntr = 0x8;
7511         } else {
7512                 /* Not supported. Apply the same limits as in the max case */
7513                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7514                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7515                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7516                 tribuf_calcntr = 0;
7517         }
7518         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7519
7520         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7521         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7522         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7523         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7524
7525         /* AFC Recal */
7526         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7527                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7528                         DPIO_AFC_RECAL);
7529
7530         mutex_unlock(&dev_priv->sb_lock);
7531 }
7532
7533 /**
7534  * vlv_force_pll_on - forcibly enable just the PLL
7535  * @dev_priv: i915 private structure
7536  * @pipe: pipe PLL to enable
7537  * @dpll: PLL configuration
7538  *
7539  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7540  * in cases where we need the PLL enabled even when @pipe is not going to
7541  * be enabled.
7542  */
7543 void vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
7544                       const struct dpll *dpll)
7545 {
7546         struct intel_crtc *crtc =
7547                 to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
7548         struct intel_crtc_state pipe_config = {
7549                 .base.crtc = &crtc->base,
7550                 .pixel_multiplier = 1,
7551                 .dpll = *dpll,
7552         };
7553
7554         if (IS_CHERRYVIEW(dev)) {
7555                 chv_compute_dpll(crtc, &pipe_config);
7556                 chv_prepare_pll(crtc, &pipe_config);
7557                 chv_enable_pll(crtc, &pipe_config);
7558         } else {
7559                 vlv_compute_dpll(crtc, &pipe_config);
7560                 vlv_prepare_pll(crtc, &pipe_config);
7561                 vlv_enable_pll(crtc, &pipe_config);
7562         }
7563 }
7564
7565 /**
7566  * vlv_force_pll_off - forcibly disable just the PLL
7567  * @dev_priv: i915 private structure
7568  * @pipe: pipe PLL to disable
7569  *
7570  * Disable the PLL for @pipe. To be used in cases where we need
7571  * the PLL enabled even when @pipe is not going to be enabled.
7572  */
7573 void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
7574 {
7575         if (IS_CHERRYVIEW(dev))
7576                 chv_disable_pll(to_i915(dev), pipe);
7577         else
7578                 vlv_disable_pll(to_i915(dev), pipe);
7579 }
7580
7581 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7582                               struct intel_crtc_state *crtc_state,
7583                               intel_clock_t *reduced_clock,
7584                               int num_connectors)
7585 {
7586         struct drm_device *dev = crtc->base.dev;
7587         struct drm_i915_private *dev_priv = dev->dev_private;
7588         u32 dpll;
7589         bool is_sdvo;
7590         struct dpll *clock = &crtc_state->dpll;
7591
7592         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7593
7594         is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7595                 intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI);
7596
7597         dpll = DPLL_VGA_MODE_DIS;
7598
7599         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
7600                 dpll |= DPLLB_MODE_LVDS;
7601         else
7602                 dpll |= DPLLB_MODE_DAC_SERIAL;
7603
7604         if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
7605                 dpll |= (crtc_state->pixel_multiplier - 1)
7606                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7607         }
7608
7609         if (is_sdvo)
7610                 dpll |= DPLL_SDVO_HIGH_SPEED;
7611
7612         if (crtc_state->has_dp_encoder)
7613                 dpll |= DPLL_SDVO_HIGH_SPEED;
7614
7615         /* compute bitmask from p1 value */
7616         if (IS_PINEVIEW(dev))
7617                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7618         else {
7619                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7620                 if (IS_G4X(dev) && reduced_clock)
7621                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7622         }
7623         switch (clock->p2) {
7624         case 5:
7625                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7626                 break;
7627         case 7:
7628                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7629                 break;
7630         case 10:
7631                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7632                 break;
7633         case 14:
7634                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7635                 break;
7636         }
7637         if (INTEL_INFO(dev)->gen >= 4)
7638                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7639
7640         if (crtc_state->sdvo_tv_clock)
7641                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7642         else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7643                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7644                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7645         else
7646                 dpll |= PLL_REF_INPUT_DREFCLK;
7647
7648         dpll |= DPLL_VCO_ENABLE;
7649         crtc_state->dpll_hw_state.dpll = dpll;
7650
7651         if (INTEL_INFO(dev)->gen >= 4) {
7652                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7653                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7654                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7655         }
7656 }
7657
7658 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7659                               struct intel_crtc_state *crtc_state,
7660                               intel_clock_t *reduced_clock,
7661                               int num_connectors)
7662 {
7663         struct drm_device *dev = crtc->base.dev;
7664         struct drm_i915_private *dev_priv = dev->dev_private;
7665         u32 dpll;
7666         struct dpll *clock = &crtc_state->dpll;
7667
7668         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7669
7670         dpll = DPLL_VGA_MODE_DIS;
7671
7672         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7673                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7674         } else {
7675                 if (clock->p1 == 2)
7676                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7677                 else
7678                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7679                 if (clock->p2 == 4)
7680                         dpll |= PLL_P2_DIVIDE_BY_4;
7681         }
7682
7683         if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
7684                 dpll |= DPLL_DVO_2X_MODE;
7685
7686         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7687                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7688                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7689         else
7690                 dpll |= PLL_REF_INPUT_DREFCLK;
7691
7692         dpll |= DPLL_VCO_ENABLE;
7693         crtc_state->dpll_hw_state.dpll = dpll;
7694 }
7695
7696 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7697 {
7698         struct drm_device *dev = intel_crtc->base.dev;
7699         struct drm_i915_private *dev_priv = dev->dev_private;
7700         enum pipe pipe = intel_crtc->pipe;
7701         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7702         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
7703         uint32_t crtc_vtotal, crtc_vblank_end;
7704         int vsyncshift = 0;
7705
7706         /* We need to be careful not to changed the adjusted mode, for otherwise
7707          * the hw state checker will get angry at the mismatch. */
7708         crtc_vtotal = adjusted_mode->crtc_vtotal;
7709         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7710
7711         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7712                 /* the chip adds 2 halflines automatically */
7713                 crtc_vtotal -= 1;
7714                 crtc_vblank_end -= 1;
7715
7716                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7717                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7718                 else
7719                         vsyncshift = adjusted_mode->crtc_hsync_start -
7720                                 adjusted_mode->crtc_htotal / 2;
7721                 if (vsyncshift < 0)
7722                         vsyncshift += adjusted_mode->crtc_htotal;
7723         }
7724
7725         if (INTEL_INFO(dev)->gen > 3)
7726                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7727
7728         I915_WRITE(HTOTAL(cpu_transcoder),
7729                    (adjusted_mode->crtc_hdisplay - 1) |
7730                    ((adjusted_mode->crtc_htotal - 1) << 16));
7731         I915_WRITE(HBLANK(cpu_transcoder),
7732                    (adjusted_mode->crtc_hblank_start - 1) |
7733                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7734         I915_WRITE(HSYNC(cpu_transcoder),
7735                    (adjusted_mode->crtc_hsync_start - 1) |
7736                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7737
7738         I915_WRITE(VTOTAL(cpu_transcoder),
7739                    (adjusted_mode->crtc_vdisplay - 1) |
7740                    ((crtc_vtotal - 1) << 16));
7741         I915_WRITE(VBLANK(cpu_transcoder),
7742                    (adjusted_mode->crtc_vblank_start - 1) |
7743                    ((crtc_vblank_end - 1) << 16));
7744         I915_WRITE(VSYNC(cpu_transcoder),
7745                    (adjusted_mode->crtc_vsync_start - 1) |
7746                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7747
7748         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7749          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7750          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7751          * bits. */
7752         if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
7753             (pipe == PIPE_B || pipe == PIPE_C))
7754                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7755
7756         /* pipesrc controls the size that is scaled from, which should
7757          * always be the user's requested size.
7758          */
7759         I915_WRITE(PIPESRC(pipe),
7760                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7761                    (intel_crtc->config->pipe_src_h - 1));
7762 }
7763
7764 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7765                                    struct intel_crtc_state *pipe_config)
7766 {
7767         struct drm_device *dev = crtc->base.dev;
7768         struct drm_i915_private *dev_priv = dev->dev_private;
7769         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7770         uint32_t tmp;
7771
7772         tmp = I915_READ(HTOTAL(cpu_transcoder));
7773         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7774         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7775         tmp = I915_READ(HBLANK(cpu_transcoder));
7776         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7777         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7778         tmp = I915_READ(HSYNC(cpu_transcoder));
7779         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7780         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7781
7782         tmp = I915_READ(VTOTAL(cpu_transcoder));
7783         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7784         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7785         tmp = I915_READ(VBLANK(cpu_transcoder));
7786         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7787         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7788         tmp = I915_READ(VSYNC(cpu_transcoder));
7789         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7790         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7791
7792         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7793                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7794                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7795                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7796         }
7797
7798         tmp = I915_READ(PIPESRC(crtc->pipe));
7799         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7800         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7801
7802         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7803         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7804 }
7805
7806 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7807                                  struct intel_crtc_state *pipe_config)
7808 {
7809         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7810         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7811         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7812         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7813
7814         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7815         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7816         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7817         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7818
7819         mode->flags = pipe_config->base.adjusted_mode.flags;
7820         mode->type = DRM_MODE_TYPE_DRIVER;
7821
7822         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7823         mode->flags |= pipe_config->base.adjusted_mode.flags;
7824
7825         mode->hsync = drm_mode_hsync(mode);
7826         mode->vrefresh = drm_mode_vrefresh(mode);
7827         drm_mode_set_name(mode);
7828 }
7829
7830 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
7831 {
7832         struct drm_device *dev = intel_crtc->base.dev;
7833         struct drm_i915_private *dev_priv = dev->dev_private;
7834         uint32_t pipeconf;
7835
7836         pipeconf = 0;
7837
7838         if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
7839             (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
7840                 pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
7841
7842         if (intel_crtc->config->double_wide)
7843                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7844
7845         /* only g4x and later have fancy bpc/dither controls */
7846         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
7847                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7848                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
7849                         pipeconf |= PIPECONF_DITHER_EN |
7850                                     PIPECONF_DITHER_TYPE_SP;
7851
7852                 switch (intel_crtc->config->pipe_bpp) {
7853                 case 18:
7854                         pipeconf |= PIPECONF_6BPC;
7855                         break;
7856                 case 24:
7857                         pipeconf |= PIPECONF_8BPC;
7858                         break;
7859                 case 30:
7860                         pipeconf |= PIPECONF_10BPC;
7861                         break;
7862                 default:
7863                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7864                         BUG();
7865                 }
7866         }
7867
7868         if (HAS_PIPE_CXSR(dev)) {
7869                 if (intel_crtc->lowfreq_avail) {
7870                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
7871                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
7872                 } else {
7873                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
7874                 }
7875         }
7876
7877         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7878                 if (INTEL_INFO(dev)->gen < 4 ||
7879                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7880                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7881                 else
7882                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7883         } else
7884                 pipeconf |= PIPECONF_PROGRESSIVE;
7885
7886         if (IS_VALLEYVIEW(dev) && intel_crtc->config->limited_color_range)
7887                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7888
7889         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
7890         POSTING_READ(PIPECONF(intel_crtc->pipe));
7891 }
7892
7893 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7894                                    struct intel_crtc_state *crtc_state)
7895 {
7896         struct drm_device *dev = crtc->base.dev;
7897         struct drm_i915_private *dev_priv = dev->dev_private;
7898         int refclk, num_connectors = 0;
7899         intel_clock_t clock;
7900         bool ok;
7901         bool is_dsi = false;
7902         struct intel_encoder *encoder;
7903         const intel_limit_t *limit;
7904         struct drm_atomic_state *state = crtc_state->base.state;
7905         struct drm_connector *connector;
7906         struct drm_connector_state *connector_state;
7907         int i;
7908
7909         memset(&crtc_state->dpll_hw_state, 0,
7910                sizeof(crtc_state->dpll_hw_state));
7911
7912         for_each_connector_in_state(state, connector, connector_state, i) {
7913                 if (connector_state->crtc != &crtc->base)
7914                         continue;
7915
7916                 encoder = to_intel_encoder(connector_state->best_encoder);
7917
7918                 switch (encoder->type) {
7919                 case INTEL_OUTPUT_DSI:
7920                         is_dsi = true;
7921                         break;
7922                 default:
7923                         break;
7924                 }
7925
7926                 num_connectors++;
7927         }
7928
7929         if (is_dsi)
7930                 return 0;
7931
7932         if (!crtc_state->clock_set) {
7933                 refclk = i9xx_get_refclk(crtc_state, num_connectors);
7934
7935                 /*
7936                  * Returns a set of divisors for the desired target clock with
7937                  * the given refclk, or FALSE.  The returned values represent
7938                  * the clock equation: reflck * (5 * (m1 + 2) + (m2 + 2)) / (n +
7939                  * 2) / p1 / p2.
7940                  */
7941                 limit = intel_limit(crtc_state, refclk);
7942                 ok = dev_priv->display.find_dpll(limit, crtc_state,
7943                                                  crtc_state->port_clock,
7944                                                  refclk, NULL, &clock);
7945                 if (!ok) {
7946                         DRM_ERROR("Couldn't find PLL settings for mode!\n");
7947                         return -EINVAL;
7948                 }
7949
7950                 /* Compat-code for transition, will disappear. */
7951                 crtc_state->dpll.n = clock.n;
7952                 crtc_state->dpll.m1 = clock.m1;
7953                 crtc_state->dpll.m2 = clock.m2;
7954                 crtc_state->dpll.p1 = clock.p1;
7955                 crtc_state->dpll.p2 = clock.p2;
7956         }
7957
7958         if (IS_GEN2(dev)) {
7959                 i8xx_compute_dpll(crtc, crtc_state, NULL,
7960                                   num_connectors);
7961         } else if (IS_CHERRYVIEW(dev)) {
7962                 chv_compute_dpll(crtc, crtc_state);
7963         } else if (IS_VALLEYVIEW(dev)) {
7964                 vlv_compute_dpll(crtc, crtc_state);
7965         } else {
7966                 i9xx_compute_dpll(crtc, crtc_state, NULL,
7967                                   num_connectors);
7968         }
7969
7970         return 0;
7971 }
7972
7973 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7974                                  struct intel_crtc_state *pipe_config)
7975 {
7976         struct drm_device *dev = crtc->base.dev;
7977         struct drm_i915_private *dev_priv = dev->dev_private;
7978         uint32_t tmp;
7979
7980         if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
7981                 return;
7982
7983         tmp = I915_READ(PFIT_CONTROL);
7984         if (!(tmp & PFIT_ENABLE))
7985                 return;
7986
7987         /* Check whether the pfit is attached to our pipe. */
7988         if (INTEL_INFO(dev)->gen < 4) {
7989                 if (crtc->pipe != PIPE_B)
7990                         return;
7991         } else {
7992                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7993                         return;
7994         }
7995
7996         pipe_config->gmch_pfit.control = tmp;
7997         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7998         if (INTEL_INFO(dev)->gen < 5)
7999                 pipe_config->gmch_pfit.lvds_border_bits =
8000                         I915_READ(LVDS) & LVDS_BORDER_ENABLE;
8001 }
8002
8003 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8004                                struct intel_crtc_state *pipe_config)
8005 {
8006         struct drm_device *dev = crtc->base.dev;
8007         struct drm_i915_private *dev_priv = dev->dev_private;
8008         int pipe = pipe_config->cpu_transcoder;
8009         intel_clock_t clock;
8010         u32 mdiv;
8011         int refclk = 100000;
8012
8013         /* In case of MIPI DPLL will not even be used */
8014         if (!(pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE))
8015                 return;
8016
8017         mutex_lock(&dev_priv->sb_lock);
8018         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8019         mutex_unlock(&dev_priv->sb_lock);
8020
8021         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8022         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8023         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8024         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8025         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8026
8027         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8028 }
8029
8030 static void
8031 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8032                               struct intel_initial_plane_config *plane_config)
8033 {
8034         struct drm_device *dev = crtc->base.dev;
8035         struct drm_i915_private *dev_priv = dev->dev_private;
8036         u32 val, base, offset;
8037         int pipe = crtc->pipe, plane = crtc->plane;
8038         int fourcc, pixel_format;
8039         unsigned int aligned_height;
8040         struct drm_framebuffer *fb;
8041         struct intel_framebuffer *intel_fb;
8042
8043         val = I915_READ(DSPCNTR(plane));
8044         if (!(val & DISPLAY_PLANE_ENABLE))
8045                 return;
8046
8047         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8048         if (!intel_fb) {
8049                 DRM_DEBUG_KMS("failed to alloc fb\n");
8050                 return;
8051         }
8052
8053         fb = &intel_fb->base;
8054
8055         if (INTEL_INFO(dev)->gen >= 4) {
8056                 if (val & DISPPLANE_TILED) {
8057                         plane_config->tiling = I915_TILING_X;
8058                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
8059                 }
8060         }
8061
8062         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8063         fourcc = i9xx_format_to_fourcc(pixel_format);
8064         fb->pixel_format = fourcc;
8065         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
8066
8067         if (INTEL_INFO(dev)->gen >= 4) {
8068                 if (plane_config->tiling)
8069                         offset = I915_READ(DSPTILEOFF(plane));
8070                 else
8071                         offset = I915_READ(DSPLINOFF(plane));
8072                 base = I915_READ(DSPSURF(plane)) & 0xfffff000;
8073         } else {
8074                 base = I915_READ(DSPADDR(plane));
8075         }
8076         plane_config->base = base;
8077
8078         val = I915_READ(PIPESRC(pipe));
8079         fb->width = ((val >> 16) & 0xfff) + 1;
8080         fb->height = ((val >> 0) & 0xfff) + 1;
8081
8082         val = I915_READ(DSPSTRIDE(pipe));
8083         fb->pitches[0] = val & 0xffffffc0;
8084
8085         aligned_height = intel_fb_align_height(dev, fb->height,
8086                                                fb->pixel_format,
8087                                                fb->modifier[0]);
8088
8089         plane_config->size = fb->pitches[0] * aligned_height;
8090
8091         DRM_DEBUG_KMS("pipe/plane %c/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8092                       pipe_name(pipe), plane, fb->width, fb->height,
8093                       fb->bits_per_pixel, base, fb->pitches[0],
8094                       plane_config->size);
8095
8096         plane_config->fb = intel_fb;
8097 }
8098
8099 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8100                                struct intel_crtc_state *pipe_config)
8101 {
8102         struct drm_device *dev = crtc->base.dev;
8103         struct drm_i915_private *dev_priv = dev->dev_private;
8104         int pipe = pipe_config->cpu_transcoder;
8105         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8106         intel_clock_t clock;
8107         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8108         int refclk = 100000;
8109
8110         mutex_lock(&dev_priv->sb_lock);
8111         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8112         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8113         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8114         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8115         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8116         mutex_unlock(&dev_priv->sb_lock);
8117
8118         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8119         clock.m2 = (pll_dw0 & 0xff) << 22;
8120         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8121                 clock.m2 |= pll_dw2 & 0x3fffff;
8122         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8123         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8124         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8125
8126         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8127 }
8128
8129 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8130                                  struct intel_crtc_state *pipe_config)
8131 {
8132         struct drm_device *dev = crtc->base.dev;
8133         struct drm_i915_private *dev_priv = dev->dev_private;
8134         uint32_t tmp;
8135
8136         if (!intel_display_power_is_enabled(dev_priv,
8137                                             POWER_DOMAIN_PIPE(crtc->pipe)))
8138                 return false;
8139
8140         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8141         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
8142
8143         tmp = I915_READ(PIPECONF(crtc->pipe));
8144         if (!(tmp & PIPECONF_ENABLE))
8145                 return false;
8146
8147         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
8148                 switch (tmp & PIPECONF_BPC_MASK) {
8149                 case PIPECONF_6BPC:
8150                         pipe_config->pipe_bpp = 18;
8151                         break;
8152                 case PIPECONF_8BPC:
8153                         pipe_config->pipe_bpp = 24;
8154                         break;
8155                 case PIPECONF_10BPC:
8156                         pipe_config->pipe_bpp = 30;
8157                         break;
8158                 default:
8159                         break;
8160                 }
8161         }
8162
8163         if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT))
8164                 pipe_config->limited_color_range = true;
8165
8166         if (INTEL_INFO(dev)->gen < 4)
8167                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8168
8169         intel_get_pipe_timings(crtc, pipe_config);
8170
8171         i9xx_get_pfit_config(crtc, pipe_config);
8172
8173         if (INTEL_INFO(dev)->gen >= 4) {
8174                 tmp = I915_READ(DPLL_MD(crtc->pipe));
8175                 pipe_config->pixel_multiplier =
8176                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8177                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8178                 pipe_config->dpll_hw_state.dpll_md = tmp;
8179         } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
8180                 tmp = I915_READ(DPLL(crtc->pipe));
8181                 pipe_config->pixel_multiplier =
8182                         ((tmp & SDVO_MULTIPLIER_MASK)
8183                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8184         } else {
8185                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8186                  * port and will be fixed up in the encoder->get_config
8187                  * function. */
8188                 pipe_config->pixel_multiplier = 1;
8189         }
8190         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8191         if (!IS_VALLEYVIEW(dev)) {
8192                 /*
8193                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8194                  * on 830. Filter it out here so that we don't
8195                  * report errors due to that.
8196                  */
8197                 if (IS_I830(dev))
8198                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8199
8200                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8201                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8202         } else {
8203                 /* Mask out read-only status bits. */
8204                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8205                                                      DPLL_PORTC_READY_MASK |
8206                                                      DPLL_PORTB_READY_MASK);
8207         }
8208
8209         if (IS_CHERRYVIEW(dev))
8210                 chv_crtc_clock_get(crtc, pipe_config);
8211         else if (IS_VALLEYVIEW(dev))
8212                 vlv_crtc_clock_get(crtc, pipe_config);
8213         else
8214                 i9xx_crtc_clock_get(crtc, pipe_config);
8215
8216         /*
8217          * Normally the dotclock is filled in by the encoder .get_config()
8218          * but in case the pipe is enabled w/o any ports we need a sane
8219          * default.
8220          */
8221         pipe_config->base.adjusted_mode.crtc_clock =
8222                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8223
8224         return true;
8225 }
8226
8227 static void ironlake_init_pch_refclk(struct drm_device *dev)
8228 {
8229         struct drm_i915_private *dev_priv = dev->dev_private;
8230         struct intel_encoder *encoder;
8231         u32 val, final;
8232         bool has_lvds = false;
8233         bool has_cpu_edp = false;
8234         bool has_panel = false;
8235         bool has_ck505 = false;
8236         bool can_ssc = false;
8237
8238         /* We need to take the global config into account */
8239         for_each_intel_encoder(dev, encoder) {
8240                 switch (encoder->type) {
8241                 case INTEL_OUTPUT_LVDS:
8242                         has_panel = true;
8243                         has_lvds = true;
8244                         break;
8245                 case INTEL_OUTPUT_EDP:
8246                         has_panel = true;
8247                         if (enc_to_dig_port(&encoder->base)->port == PORT_A)
8248                                 has_cpu_edp = true;
8249                         break;
8250                 default:
8251                         break;
8252                 }
8253         }
8254
8255         if (HAS_PCH_IBX(dev)) {
8256                 has_ck505 = dev_priv->vbt.display_clock_mode;
8257                 can_ssc = has_ck505;
8258         } else {
8259                 has_ck505 = false;
8260                 can_ssc = true;
8261         }
8262
8263         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d\n",
8264                       has_panel, has_lvds, has_ck505);
8265
8266         /* Ironlake: try to setup display ref clock before DPLL
8267          * enabling. This is only under driver's control after
8268          * PCH B stepping, previous chipset stepping should be
8269          * ignoring this setting.
8270          */
8271         val = I915_READ(PCH_DREF_CONTROL);
8272
8273         /* As we must carefully and slowly disable/enable each source in turn,
8274          * compute the final state we want first and check if we need to
8275          * make any changes at all.
8276          */
8277         final = val;
8278         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8279         if (has_ck505)
8280                 final |= DREF_NONSPREAD_CK505_ENABLE;
8281         else
8282                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8283
8284         final &= ~DREF_SSC_SOURCE_MASK;
8285         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8286         final &= ~DREF_SSC1_ENABLE;
8287
8288         if (has_panel) {
8289                 final |= DREF_SSC_SOURCE_ENABLE;
8290
8291                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8292                         final |= DREF_SSC1_ENABLE;
8293
8294                 if (has_cpu_edp) {
8295                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8296                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8297                         else
8298                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8299                 } else
8300                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8301         } else {
8302                 final |= DREF_SSC_SOURCE_DISABLE;
8303                 final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8304         }
8305
8306         if (final == val)
8307                 return;
8308
8309         /* Always enable nonspread source */
8310         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8311
8312         if (has_ck505)
8313                 val |= DREF_NONSPREAD_CK505_ENABLE;
8314         else
8315                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8316
8317         if (has_panel) {
8318                 val &= ~DREF_SSC_SOURCE_MASK;
8319                 val |= DREF_SSC_SOURCE_ENABLE;
8320
8321                 /* SSC must be turned on before enabling the CPU output  */
8322                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8323                         DRM_DEBUG_KMS("Using SSC on panel\n");
8324                         val |= DREF_SSC1_ENABLE;
8325                 } else
8326                         val &= ~DREF_SSC1_ENABLE;
8327
8328                 /* Get SSC going before enabling the outputs */
8329                 I915_WRITE(PCH_DREF_CONTROL, val);
8330                 POSTING_READ(PCH_DREF_CONTROL);
8331                 udelay(200);
8332
8333                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8334
8335                 /* Enable CPU source on CPU attached eDP */
8336                 if (has_cpu_edp) {
8337                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8338                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8339                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8340                         } else
8341                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8342                 } else
8343                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8344
8345                 I915_WRITE(PCH_DREF_CONTROL, val);
8346                 POSTING_READ(PCH_DREF_CONTROL);
8347                 udelay(200);
8348         } else {
8349                 DRM_DEBUG_KMS("Disabling SSC entirely\n");
8350
8351                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8352
8353                 /* Turn off CPU output */
8354                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8355
8356                 I915_WRITE(PCH_DREF_CONTROL, val);
8357                 POSTING_READ(PCH_DREF_CONTROL);
8358                 udelay(200);
8359
8360                 /* Turn off the SSC source */
8361                 val &= ~DREF_SSC_SOURCE_MASK;
8362                 val |= DREF_SSC_SOURCE_DISABLE;
8363
8364                 /* Turn off SSC1 */
8365                 val &= ~DREF_SSC1_ENABLE;
8366
8367                 I915_WRITE(PCH_DREF_CONTROL, val);
8368                 POSTING_READ(PCH_DREF_CONTROL);
8369                 udelay(200);
8370         }
8371
8372         BUG_ON(val != final);
8373 }
8374
8375 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8376 {
8377         uint32_t tmp;
8378
8379         tmp = I915_READ(SOUTH_CHICKEN2);
8380         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8381         I915_WRITE(SOUTH_CHICKEN2, tmp);
8382
8383         if (wait_for_atomic_us(I915_READ(SOUTH_CHICKEN2) &
8384                                FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8385                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8386
8387         tmp = I915_READ(SOUTH_CHICKEN2);
8388         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8389         I915_WRITE(SOUTH_CHICKEN2, tmp);
8390
8391         if (wait_for_atomic_us((I915_READ(SOUTH_CHICKEN2) &
8392                                 FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8393                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8394 }
8395
8396 /* WaMPhyProgramming:hsw */
8397 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8398 {
8399         uint32_t tmp;
8400
8401         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8402         tmp &= ~(0xFF << 24);
8403         tmp |= (0x12 << 24);
8404         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8405
8406         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8407         tmp |= (1 << 11);
8408         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8409
8410         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8411         tmp |= (1 << 11);
8412         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8413
8414         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8415         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8416         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8417
8418         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8419         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8420         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8421
8422         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8423         tmp &= ~(7 << 13);
8424         tmp |= (5 << 13);
8425         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8426
8427         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8428         tmp &= ~(7 << 13);
8429         tmp |= (5 << 13);
8430         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8431
8432         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8433         tmp &= ~0xFF;
8434         tmp |= 0x1C;
8435         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8436
8437         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8438         tmp &= ~0xFF;
8439         tmp |= 0x1C;
8440         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8441
8442         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8443         tmp &= ~(0xFF << 16);
8444         tmp |= (0x1C << 16);
8445         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8446
8447         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8448         tmp &= ~(0xFF << 16);
8449         tmp |= (0x1C << 16);
8450         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8451
8452         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8453         tmp |= (1 << 27);
8454         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8455
8456         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8457         tmp |= (1 << 27);
8458         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8459
8460         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8461         tmp &= ~(0xF << 28);
8462         tmp |= (4 << 28);
8463         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8464
8465         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8466         tmp &= ~(0xF << 28);
8467         tmp |= (4 << 28);
8468         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8469 }
8470
8471 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8472  * Programming" based on the parameters passed:
8473  * - Sequence to enable CLKOUT_DP
8474  * - Sequence to enable CLKOUT_DP without spread
8475  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8476  */
8477 static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
8478                                  bool with_fdi)
8479 {
8480         struct drm_i915_private *dev_priv = dev->dev_private;
8481         uint32_t reg, tmp;
8482
8483         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8484                 with_spread = true;
8485         if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
8486                 with_fdi = false;
8487
8488         mutex_lock(&dev_priv->sb_lock);
8489
8490         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8491         tmp &= ~SBI_SSCCTL_DISABLE;
8492         tmp |= SBI_SSCCTL_PATHALT;
8493         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8494
8495         udelay(24);
8496
8497         if (with_spread) {
8498                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8499                 tmp &= ~SBI_SSCCTL_PATHALT;
8500                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8501
8502                 if (with_fdi) {
8503                         lpt_reset_fdi_mphy(dev_priv);
8504                         lpt_program_fdi_mphy(dev_priv);
8505                 }
8506         }
8507
8508         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8509         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8510         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8511         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8512
8513         mutex_unlock(&dev_priv->sb_lock);
8514 }
8515
8516 /* Sequence to disable CLKOUT_DP */
8517 static void lpt_disable_clkout_dp(struct drm_device *dev)
8518 {
8519         struct drm_i915_private *dev_priv = dev->dev_private;
8520         uint32_t reg, tmp;
8521
8522         mutex_lock(&dev_priv->sb_lock);
8523
8524         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8525         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8526         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8527         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8528
8529         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8530         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8531                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8532                         tmp |= SBI_SSCCTL_PATHALT;
8533                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8534                         udelay(32);
8535                 }
8536                 tmp |= SBI_SSCCTL_DISABLE;
8537                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8538         }
8539
8540         mutex_unlock(&dev_priv->sb_lock);
8541 }
8542
8543 static void lpt_init_pch_refclk(struct drm_device *dev)
8544 {
8545         struct intel_encoder *encoder;
8546         bool has_vga = false;
8547
8548         for_each_intel_encoder(dev, encoder) {
8549                 switch (encoder->type) {
8550                 case INTEL_OUTPUT_ANALOG:
8551                         has_vga = true;
8552                         break;
8553                 default:
8554                         break;
8555                 }
8556         }
8557
8558         if (has_vga)
8559                 lpt_enable_clkout_dp(dev, true, true);
8560         else
8561                 lpt_disable_clkout_dp(dev);
8562 }
8563
8564 /*
8565  * Initialize reference clocks when the driver loads
8566  */
8567 void intel_init_pch_refclk(struct drm_device *dev)
8568 {
8569         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
8570                 ironlake_init_pch_refclk(dev);
8571         else if (HAS_PCH_LPT(dev))
8572                 lpt_init_pch_refclk(dev);
8573 }
8574
8575 static int ironlake_get_refclk(struct intel_crtc_state *crtc_state)
8576 {
8577         struct drm_device *dev = crtc_state->base.crtc->dev;
8578         struct drm_i915_private *dev_priv = dev->dev_private;
8579         struct drm_atomic_state *state = crtc_state->base.state;
8580         struct drm_connector *connector;
8581         struct drm_connector_state *connector_state;
8582         struct intel_encoder *encoder;
8583         int num_connectors = 0, i;
8584         bool is_lvds = false;
8585
8586         for_each_connector_in_state(state, connector, connector_state, i) {
8587                 if (connector_state->crtc != crtc_state->base.crtc)
8588                         continue;
8589
8590                 encoder = to_intel_encoder(connector_state->best_encoder);
8591
8592                 switch (encoder->type) {
8593                 case INTEL_OUTPUT_LVDS:
8594                         is_lvds = true;
8595                         break;
8596                 default:
8597                         break;
8598                 }
8599                 num_connectors++;
8600         }
8601
8602         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
8603                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8604                               dev_priv->vbt.lvds_ssc_freq);
8605                 return dev_priv->vbt.lvds_ssc_freq;
8606         }
8607
8608         return 120000;
8609 }
8610
8611 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
8612 {
8613         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8614         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8615         int pipe = intel_crtc->pipe;
8616         uint32_t val;
8617
8618         val = 0;
8619
8620         switch (intel_crtc->config->pipe_bpp) {
8621         case 18:
8622                 val |= PIPECONF_6BPC;
8623                 break;
8624         case 24:
8625                 val |= PIPECONF_8BPC;
8626                 break;
8627         case 30:
8628                 val |= PIPECONF_10BPC;
8629                 break;
8630         case 36:
8631                 val |= PIPECONF_12BPC;
8632                 break;
8633         default:
8634                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8635                 BUG();
8636         }
8637
8638         if (intel_crtc->config->dither)
8639                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8640
8641         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8642                 val |= PIPECONF_INTERLACED_ILK;
8643         else
8644                 val |= PIPECONF_PROGRESSIVE;
8645
8646         if (intel_crtc->config->limited_color_range)
8647                 val |= PIPECONF_COLOR_RANGE_SELECT;
8648
8649         I915_WRITE(PIPECONF(pipe), val);
8650         POSTING_READ(PIPECONF(pipe));
8651 }
8652
8653 /*
8654  * Set up the pipe CSC unit.
8655  *
8656  * Currently only full range RGB to limited range RGB conversion
8657  * is supported, but eventually this should handle various
8658  * RGB<->YCbCr scenarios as well.
8659  */
8660 static void intel_set_pipe_csc(struct drm_crtc *crtc)
8661 {
8662         struct drm_device *dev = crtc->dev;
8663         struct drm_i915_private *dev_priv = dev->dev_private;
8664         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8665         int pipe = intel_crtc->pipe;
8666         uint16_t coeff = 0x7800; /* 1.0 */
8667
8668         /*
8669          * TODO: Check what kind of values actually come out of the pipe
8670          * with these coeff/postoff values and adjust to get the best
8671          * accuracy. Perhaps we even need to take the bpc value into
8672          * consideration.
8673          */
8674
8675         if (intel_crtc->config->limited_color_range)
8676                 coeff = ((235 - 16) * (1 << 12) / 255) & 0xff8; /* 0.xxx... */
8677
8678         /*
8679          * GY/GU and RY/RU should be the other way around according
8680          * to BSpec, but reality doesn't agree. Just set them up in
8681          * a way that results in the correct picture.
8682          */
8683         I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeff << 16);
8684         I915_WRITE(PIPE_CSC_COEFF_BY(pipe), 0);
8685
8686         I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeff);
8687         I915_WRITE(PIPE_CSC_COEFF_BU(pipe), 0);
8688
8689         I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), 0);
8690         I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeff << 16);
8691
8692         I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
8693         I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
8694         I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
8695
8696         if (INTEL_INFO(dev)->gen > 6) {
8697                 uint16_t postoff = 0;
8698
8699                 if (intel_crtc->config->limited_color_range)
8700                         postoff = (16 * (1 << 12) / 255) & 0x1fff;
8701
8702                 I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
8703                 I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff);
8704                 I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff);
8705
8706                 I915_WRITE(PIPE_CSC_MODE(pipe), 0);
8707         } else {
8708                 uint32_t mode = CSC_MODE_YUV_TO_RGB;
8709
8710                 if (intel_crtc->config->limited_color_range)
8711                         mode |= CSC_BLACK_SCREEN_OFFSET;
8712
8713                 I915_WRITE(PIPE_CSC_MODE(pipe), mode);
8714         }
8715 }
8716
8717 static void haswell_set_pipeconf(struct drm_crtc *crtc)
8718 {
8719         struct drm_device *dev = crtc->dev;
8720         struct drm_i915_private *dev_priv = dev->dev_private;
8721         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8722         enum pipe pipe = intel_crtc->pipe;
8723         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8724         uint32_t val;
8725
8726         val = 0;
8727
8728         if (IS_HASWELL(dev) && intel_crtc->config->dither)
8729                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8730
8731         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8732                 val |= PIPECONF_INTERLACED_ILK;
8733         else
8734                 val |= PIPECONF_PROGRESSIVE;
8735
8736         I915_WRITE(PIPECONF(cpu_transcoder), val);
8737         POSTING_READ(PIPECONF(cpu_transcoder));
8738
8739         I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT);
8740         POSTING_READ(GAMMA_MODE(intel_crtc->pipe));
8741
8742         if (IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9) {
8743                 val = 0;
8744
8745                 switch (intel_crtc->config->pipe_bpp) {
8746                 case 18:
8747                         val |= PIPEMISC_DITHER_6_BPC;
8748                         break;
8749                 case 24:
8750                         val |= PIPEMISC_DITHER_8_BPC;
8751                         break;
8752                 case 30:
8753                         val |= PIPEMISC_DITHER_10_BPC;
8754                         break;
8755                 case 36:
8756                         val |= PIPEMISC_DITHER_12_BPC;
8757                         break;
8758                 default:
8759                         /* Case prevented by pipe_config_set_bpp. */
8760                         BUG();
8761                 }
8762
8763                 if (intel_crtc->config->dither)
8764                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8765
8766                 I915_WRITE(PIPEMISC(pipe), val);
8767         }
8768 }
8769
8770 static bool ironlake_compute_clocks(struct drm_crtc *crtc,
8771                                     struct intel_crtc_state *crtc_state,
8772                                     intel_clock_t *clock,
8773                                     bool *has_reduced_clock,
8774                                     intel_clock_t *reduced_clock)
8775 {
8776         struct drm_device *dev = crtc->dev;
8777         struct drm_i915_private *dev_priv = dev->dev_private;
8778         int refclk;
8779         const intel_limit_t *limit;
8780         bool ret;
8781
8782         refclk = ironlake_get_refclk(crtc_state);
8783
8784         /*
8785          * Returns a set of divisors for the desired target clock with the given
8786          * refclk, or FALSE.  The returned values represent the clock equation:
8787          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
8788          */
8789         limit = intel_limit(crtc_state, refclk);
8790         ret = dev_priv->display.find_dpll(limit, crtc_state,
8791                                           crtc_state->port_clock,
8792                                           refclk, NULL, clock);
8793         if (!ret)
8794                 return false;
8795
8796         return true;
8797 }
8798
8799 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8800 {
8801         /*
8802          * Account for spread spectrum to avoid
8803          * oversubscribing the link. Max center spread
8804          * is 2.5%; use 5% for safety's sake.
8805          */
8806         u32 bps = target_clock * bpp * 21 / 20;
8807         return DIV_ROUND_UP(bps, link_bw * 8);
8808 }
8809
8810 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8811 {
8812         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8813 }
8814
8815 static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8816                                       struct intel_crtc_state *crtc_state,
8817                                       u32 *fp,
8818                                       intel_clock_t *reduced_clock, u32 *fp2)
8819 {
8820         struct drm_crtc *crtc = &intel_crtc->base;
8821         struct drm_device *dev = crtc->dev;
8822         struct drm_i915_private *dev_priv = dev->dev_private;
8823         struct drm_atomic_state *state = crtc_state->base.state;
8824         struct drm_connector *connector;
8825         struct drm_connector_state *connector_state;
8826         struct intel_encoder *encoder;
8827         uint32_t dpll;
8828         int factor, num_connectors = 0, i;
8829         bool is_lvds = false, is_sdvo = false;
8830
8831         for_each_connector_in_state(state, connector, connector_state, i) {
8832                 if (connector_state->crtc != crtc_state->base.crtc)
8833                         continue;
8834
8835                 encoder = to_intel_encoder(connector_state->best_encoder);
8836
8837                 switch (encoder->type) {
8838                 case INTEL_OUTPUT_LVDS:
8839                         is_lvds = true;
8840                         break;
8841                 case INTEL_OUTPUT_SDVO:
8842                 case INTEL_OUTPUT_HDMI:
8843                         is_sdvo = true;
8844                         break;
8845                 default:
8846                         break;
8847                 }
8848
8849                 num_connectors++;
8850         }
8851
8852         /* Enable autotuning of the PLL clock (if permissible) */
8853         factor = 21;
8854         if (is_lvds) {
8855                 if ((intel_panel_use_ssc(dev_priv) &&
8856                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8857                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
8858                         factor = 25;
8859         } else if (crtc_state->sdvo_tv_clock)
8860                 factor = 20;
8861
8862         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8863                 *fp |= FP_CB_TUNE;
8864
8865         if (fp2 && (reduced_clock->m < factor * reduced_clock->n))
8866                 *fp2 |= FP_CB_TUNE;
8867
8868         dpll = 0;
8869
8870         if (is_lvds)
8871                 dpll |= DPLLB_MODE_LVDS;
8872         else
8873                 dpll |= DPLLB_MODE_DAC_SERIAL;
8874
8875         dpll |= (crtc_state->pixel_multiplier - 1)
8876                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8877
8878         if (is_sdvo)
8879                 dpll |= DPLL_SDVO_HIGH_SPEED;
8880         if (crtc_state->has_dp_encoder)
8881                 dpll |= DPLL_SDVO_HIGH_SPEED;
8882
8883         /* compute bitmask from p1 value */
8884         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8885         /* also FPA1 */
8886         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8887
8888         switch (crtc_state->dpll.p2) {
8889         case 5:
8890                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8891                 break;
8892         case 7:
8893                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8894                 break;
8895         case 10:
8896                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8897                 break;
8898         case 14:
8899                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8900                 break;
8901         }
8902
8903         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
8904                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8905         else
8906                 dpll |= PLL_REF_INPUT_DREFCLK;
8907
8908         return dpll | DPLL_VCO_ENABLE;
8909 }
8910
8911 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8912                                        struct intel_crtc_state *crtc_state)
8913 {
8914         struct drm_device *dev = crtc->base.dev;
8915         intel_clock_t clock, reduced_clock;
8916         u32 dpll = 0, fp = 0, fp2 = 0;
8917         bool ok, has_reduced_clock = false;
8918         bool is_lvds = false;
8919         struct intel_shared_dpll *pll;
8920
8921         memset(&crtc_state->dpll_hw_state, 0,
8922                sizeof(crtc_state->dpll_hw_state));
8923
8924         is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS);
8925
8926         WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
8927              "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
8928
8929         ok = ironlake_compute_clocks(&crtc->base, crtc_state, &clock,
8930                                      &has_reduced_clock, &reduced_clock);
8931         if (!ok && !crtc_state->clock_set) {
8932                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8933                 return -EINVAL;
8934         }
8935         /* Compat-code for transition, will disappear. */
8936         if (!crtc_state->clock_set) {
8937                 crtc_state->dpll.n = clock.n;
8938                 crtc_state->dpll.m1 = clock.m1;
8939                 crtc_state->dpll.m2 = clock.m2;
8940                 crtc_state->dpll.p1 = clock.p1;
8941                 crtc_state->dpll.p2 = clock.p2;
8942         }
8943
8944         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8945         if (crtc_state->has_pch_encoder) {
8946                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8947                 if (has_reduced_clock)
8948                         fp2 = i9xx_dpll_compute_fp(&reduced_clock);
8949
8950                 dpll = ironlake_compute_dpll(crtc, crtc_state,
8951                                              &fp, &reduced_clock,
8952                                              has_reduced_clock ? &fp2 : NULL);
8953
8954                 crtc_state->dpll_hw_state.dpll = dpll;
8955                 crtc_state->dpll_hw_state.fp0 = fp;
8956                 if (has_reduced_clock)
8957                         crtc_state->dpll_hw_state.fp1 = fp2;
8958                 else
8959                         crtc_state->dpll_hw_state.fp1 = fp;
8960
8961                 pll = intel_get_shared_dpll(crtc, crtc_state);
8962                 if (pll == NULL) {
8963                         DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
8964                                          pipe_name(crtc->pipe));
8965                         return -EINVAL;
8966                 }
8967         }
8968
8969         if (is_lvds && has_reduced_clock)
8970                 crtc->lowfreq_avail = true;
8971         else
8972                 crtc->lowfreq_avail = false;
8973
8974         return 0;
8975 }
8976
8977 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8978                                          struct intel_link_m_n *m_n)
8979 {
8980         struct drm_device *dev = crtc->base.dev;
8981         struct drm_i915_private *dev_priv = dev->dev_private;
8982         enum pipe pipe = crtc->pipe;
8983
8984         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8985         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8986         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8987                 & ~TU_SIZE_MASK;
8988         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8989         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8990                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8991 }
8992
8993 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8994                                          enum transcoder transcoder,
8995                                          struct intel_link_m_n *m_n,
8996                                          struct intel_link_m_n *m2_n2)
8997 {
8998         struct drm_device *dev = crtc->base.dev;
8999         struct drm_i915_private *dev_priv = dev->dev_private;
9000         enum pipe pipe = crtc->pipe;
9001
9002         if (INTEL_INFO(dev)->gen >= 5) {
9003                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9004                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9005                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9006                         & ~TU_SIZE_MASK;
9007                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9008                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9009                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9010                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
9011                  * gen < 8) and if DRRS is supported (to make sure the
9012                  * registers are not unnecessarily read).
9013                  */
9014                 if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
9015                         crtc->config->has_drrs) {
9016                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9017                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9018                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9019                                         & ~TU_SIZE_MASK;
9020                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9021                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9022                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9023                 }
9024         } else {
9025                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9026                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9027                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9028                         & ~TU_SIZE_MASK;
9029                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9030                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9031                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9032         }
9033 }
9034
9035 void intel_dp_get_m_n(struct intel_crtc *crtc,
9036                       struct intel_crtc_state *pipe_config)
9037 {
9038         if (pipe_config->has_pch_encoder)
9039                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9040         else
9041                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9042                                              &pipe_config->dp_m_n,
9043                                              &pipe_config->dp_m2_n2);
9044 }
9045
9046 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9047                                         struct intel_crtc_state *pipe_config)
9048 {
9049         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9050                                      &pipe_config->fdi_m_n, NULL);
9051 }
9052
9053 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9054                                     struct intel_crtc_state *pipe_config)
9055 {
9056         struct drm_device *dev = crtc->base.dev;
9057         struct drm_i915_private *dev_priv = dev->dev_private;
9058         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9059         uint32_t ps_ctrl = 0;
9060         int id = -1;
9061         int i;
9062
9063         /* find scaler attached to this pipe */
9064         for (i = 0; i < crtc->num_scalers; i++) {
9065                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9066                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9067                         id = i;
9068                         pipe_config->pch_pfit.enabled = true;
9069                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9070                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9071                         break;
9072                 }
9073         }
9074
9075         scaler_state->scaler_id = id;
9076         if (id >= 0) {
9077                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9078         } else {
9079                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9080         }
9081 }
9082
9083 static void
9084 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9085                                  struct intel_initial_plane_config *plane_config)
9086 {
9087         struct drm_device *dev = crtc->base.dev;
9088         struct drm_i915_private *dev_priv = dev->dev_private;
9089         u32 val, base, offset, stride_mult, tiling;
9090         int pipe = crtc->pipe;
9091         int fourcc, pixel_format;
9092         unsigned int aligned_height;
9093         struct drm_framebuffer *fb;
9094         struct intel_framebuffer *intel_fb;
9095
9096         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9097         if (!intel_fb) {
9098                 DRM_DEBUG_KMS("failed to alloc fb\n");
9099                 return;
9100         }
9101
9102         fb = &intel_fb->base;
9103
9104         val = I915_READ(PLANE_CTL(pipe, 0));
9105         if (!(val & PLANE_CTL_ENABLE))
9106                 goto error;
9107
9108         pixel_format = val & PLANE_CTL_FORMAT_MASK;
9109         fourcc = skl_format_to_fourcc(pixel_format,
9110                                       val & PLANE_CTL_ORDER_RGBX,
9111                                       val & PLANE_CTL_ALPHA_MASK);
9112         fb->pixel_format = fourcc;
9113         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9114
9115         tiling = val & PLANE_CTL_TILED_MASK;
9116         switch (tiling) {
9117         case PLANE_CTL_TILED_LINEAR:
9118                 fb->modifier[0] = DRM_FORMAT_MOD_NONE;
9119                 break;
9120         case PLANE_CTL_TILED_X:
9121                 plane_config->tiling = I915_TILING_X;
9122                 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9123                 break;
9124         case PLANE_CTL_TILED_Y:
9125                 fb->modifier[0] = I915_FORMAT_MOD_Y_TILED;
9126                 break;
9127         case PLANE_CTL_TILED_YF:
9128                 fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED;
9129                 break;
9130         default:
9131                 MISSING_CASE(tiling);
9132                 goto error;
9133         }
9134
9135         base = I915_READ(PLANE_SURF(pipe, 0)) & 0xfffff000;
9136         plane_config->base = base;
9137
9138         offset = I915_READ(PLANE_OFFSET(pipe, 0));
9139
9140         val = I915_READ(PLANE_SIZE(pipe, 0));
9141         fb->height = ((val >> 16) & 0xfff) + 1;
9142         fb->width = ((val >> 0) & 0x1fff) + 1;
9143
9144         val = I915_READ(PLANE_STRIDE(pipe, 0));
9145         stride_mult = intel_fb_stride_alignment(dev, fb->modifier[0],
9146                                                 fb->pixel_format);
9147         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9148
9149         aligned_height = intel_fb_align_height(dev, fb->height,
9150                                                fb->pixel_format,
9151                                                fb->modifier[0]);
9152
9153         plane_config->size = fb->pitches[0] * aligned_height;
9154
9155         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9156                       pipe_name(pipe), fb->width, fb->height,
9157                       fb->bits_per_pixel, base, fb->pitches[0],
9158                       plane_config->size);
9159
9160         plane_config->fb = intel_fb;
9161         return;
9162
9163 error:
9164         kfree(fb);
9165 }
9166
9167 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9168                                      struct intel_crtc_state *pipe_config)
9169 {
9170         struct drm_device *dev = crtc->base.dev;
9171         struct drm_i915_private *dev_priv = dev->dev_private;
9172         uint32_t tmp;
9173
9174         tmp = I915_READ(PF_CTL(crtc->pipe));
9175
9176         if (tmp & PF_ENABLE) {
9177                 pipe_config->pch_pfit.enabled = true;
9178                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9179                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9180
9181                 /* We currently do not free assignements of panel fitters on
9182                  * ivb/hsw (since we don't use the higher upscaling modes which
9183                  * differentiates them) so just WARN about this case for now. */
9184                 if (IS_GEN7(dev)) {
9185                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9186                                 PF_PIPE_SEL_IVB(crtc->pipe));
9187                 }
9188         }
9189 }
9190
9191 static void
9192 ironlake_get_initial_plane_config(struct intel_crtc *crtc,
9193                                   struct intel_initial_plane_config *plane_config)
9194 {
9195         struct drm_device *dev = crtc->base.dev;
9196         struct drm_i915_private *dev_priv = dev->dev_private;
9197         u32 val, base, offset;
9198         int pipe = crtc->pipe;
9199         int fourcc, pixel_format;
9200         unsigned int aligned_height;
9201         struct drm_framebuffer *fb;
9202         struct intel_framebuffer *intel_fb;
9203
9204         val = I915_READ(DSPCNTR(pipe));
9205         if (!(val & DISPLAY_PLANE_ENABLE))
9206                 return;
9207
9208         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9209         if (!intel_fb) {
9210                 DRM_DEBUG_KMS("failed to alloc fb\n");
9211                 return;
9212         }
9213
9214         fb = &intel_fb->base;
9215
9216         if (INTEL_INFO(dev)->gen >= 4) {
9217                 if (val & DISPPLANE_TILED) {
9218                         plane_config->tiling = I915_TILING_X;
9219                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9220                 }
9221         }
9222
9223         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
9224         fourcc = i9xx_format_to_fourcc(pixel_format);
9225         fb->pixel_format = fourcc;
9226         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9227
9228         base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
9229         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
9230                 offset = I915_READ(DSPOFFSET(pipe));
9231         } else {
9232                 if (plane_config->tiling)
9233                         offset = I915_READ(DSPTILEOFF(pipe));
9234                 else
9235                         offset = I915_READ(DSPLINOFF(pipe));
9236         }
9237         plane_config->base = base;
9238
9239         val = I915_READ(PIPESRC(pipe));
9240         fb->width = ((val >> 16) & 0xfff) + 1;
9241         fb->height = ((val >> 0) & 0xfff) + 1;
9242
9243         val = I915_READ(DSPSTRIDE(pipe));
9244         fb->pitches[0] = val & 0xffffffc0;
9245
9246         aligned_height = intel_fb_align_height(dev, fb->height,
9247                                                fb->pixel_format,
9248                                                fb->modifier[0]);
9249
9250         plane_config->size = fb->pitches[0] * aligned_height;
9251
9252         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9253                       pipe_name(pipe), fb->width, fb->height,
9254                       fb->bits_per_pixel, base, fb->pitches[0],
9255                       plane_config->size);
9256
9257         plane_config->fb = intel_fb;
9258 }
9259
9260 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9261                                      struct intel_crtc_state *pipe_config)
9262 {
9263         struct drm_device *dev = crtc->base.dev;
9264         struct drm_i915_private *dev_priv = dev->dev_private;
9265         uint32_t tmp;
9266
9267         if (!intel_display_power_is_enabled(dev_priv,
9268                                             POWER_DOMAIN_PIPE(crtc->pipe)))
9269                 return false;
9270
9271         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9272         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9273
9274         tmp = I915_READ(PIPECONF(crtc->pipe));
9275         if (!(tmp & PIPECONF_ENABLE))
9276                 return false;
9277
9278         switch (tmp & PIPECONF_BPC_MASK) {
9279         case PIPECONF_6BPC:
9280                 pipe_config->pipe_bpp = 18;
9281                 break;
9282         case PIPECONF_8BPC:
9283                 pipe_config->pipe_bpp = 24;
9284                 break;
9285         case PIPECONF_10BPC:
9286                 pipe_config->pipe_bpp = 30;
9287                 break;
9288         case PIPECONF_12BPC:
9289                 pipe_config->pipe_bpp = 36;
9290                 break;
9291         default:
9292                 break;
9293         }
9294
9295         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9296                 pipe_config->limited_color_range = true;
9297
9298         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9299                 struct intel_shared_dpll *pll;
9300
9301                 pipe_config->has_pch_encoder = true;
9302
9303                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9304                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9305                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9306
9307                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9308
9309                 if (HAS_PCH_IBX(dev_priv->dev)) {
9310                         pipe_config->shared_dpll =
9311                                 (enum intel_dpll_id) crtc->pipe;
9312                 } else {
9313                         tmp = I915_READ(PCH_DPLL_SEL);
9314                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9315                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_B;
9316                         else
9317                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_A;
9318                 }
9319
9320                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9321
9322                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9323                                            &pipe_config->dpll_hw_state));
9324
9325                 tmp = pipe_config->dpll_hw_state.dpll;
9326                 pipe_config->pixel_multiplier =
9327                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9328                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9329
9330                 ironlake_pch_clock_get(crtc, pipe_config);
9331         } else {
9332                 pipe_config->pixel_multiplier = 1;
9333         }
9334
9335         intel_get_pipe_timings(crtc, pipe_config);
9336
9337         ironlake_get_pfit_config(crtc, pipe_config);
9338
9339         return true;
9340 }
9341
9342 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9343 {
9344         struct drm_device *dev = dev_priv->dev;
9345         struct intel_crtc *crtc;
9346
9347         for_each_intel_crtc(dev, crtc)
9348                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9349                      pipe_name(crtc->pipe));
9350
9351         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
9352         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9353         I915_STATE_WARN(I915_READ(WRPLL_CTL1) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9354         I915_STATE_WARN(I915_READ(WRPLL_CTL2) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9355         I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
9356         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9357              "CPU PWM1 enabled\n");
9358         if (IS_HASWELL(dev))
9359                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9360                      "CPU PWM2 enabled\n");
9361         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9362              "PCH PWM1 enabled\n");
9363         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9364              "Utility pin enabled\n");
9365         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9366
9367         /*
9368          * In theory we can still leave IRQs enabled, as long as only the HPD
9369          * interrupts remain enabled. We used to check for that, but since it's
9370          * gen-specific and since we only disable LCPLL after we fully disable
9371          * the interrupts, the check below should be enough.
9372          */
9373         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9374 }
9375
9376 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9377 {
9378         struct drm_device *dev = dev_priv->dev;
9379
9380         if (IS_HASWELL(dev))
9381                 return I915_READ(D_COMP_HSW);
9382         else
9383                 return I915_READ(D_COMP_BDW);
9384 }
9385
9386 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9387 {
9388         struct drm_device *dev = dev_priv->dev;
9389
9390         if (IS_HASWELL(dev)) {
9391                 mutex_lock(&dev_priv->rps.hw_lock);
9392                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9393                                             val))
9394                         DRM_ERROR("Failed to write to D_COMP\n");
9395                 mutex_unlock(&dev_priv->rps.hw_lock);
9396         } else {
9397                 I915_WRITE(D_COMP_BDW, val);
9398                 POSTING_READ(D_COMP_BDW);
9399         }
9400 }
9401
9402 /*
9403  * This function implements pieces of two sequences from BSpec:
9404  * - Sequence for display software to disable LCPLL
9405  * - Sequence for display software to allow package C8+
9406  * The steps implemented here are just the steps that actually touch the LCPLL
9407  * register. Callers should take care of disabling all the display engine
9408  * functions, doing the mode unset, fixing interrupts, etc.
9409  */
9410 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9411                               bool switch_to_fclk, bool allow_power_down)
9412 {
9413         uint32_t val;
9414
9415         assert_can_disable_lcpll(dev_priv);
9416
9417         val = I915_READ(LCPLL_CTL);
9418
9419         if (switch_to_fclk) {
9420                 val |= LCPLL_CD_SOURCE_FCLK;
9421                 I915_WRITE(LCPLL_CTL, val);
9422
9423                 if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9424                                        LCPLL_CD_SOURCE_FCLK_DONE, 1))
9425                         DRM_ERROR("Switching to FCLK failed\n");
9426
9427                 val = I915_READ(LCPLL_CTL);
9428         }
9429
9430         val |= LCPLL_PLL_DISABLE;
9431         I915_WRITE(LCPLL_CTL, val);
9432         POSTING_READ(LCPLL_CTL);
9433
9434         if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1))
9435                 DRM_ERROR("LCPLL still locked\n");
9436
9437         val = hsw_read_dcomp(dev_priv);
9438         val |= D_COMP_COMP_DISABLE;
9439         hsw_write_dcomp(dev_priv, val);
9440         ndelay(100);
9441
9442         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9443                      1))
9444                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9445
9446         if (allow_power_down) {
9447                 val = I915_READ(LCPLL_CTL);
9448                 val |= LCPLL_POWER_DOWN_ALLOW;
9449                 I915_WRITE(LCPLL_CTL, val);
9450                 POSTING_READ(LCPLL_CTL);
9451         }
9452 }
9453
9454 /*
9455  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9456  * source.
9457  */
9458 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9459 {
9460         uint32_t val;
9461
9462         val = I915_READ(LCPLL_CTL);
9463
9464         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9465                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9466                 return;
9467
9468         /*
9469          * Make sure we're not on PC8 state before disabling PC8, otherwise
9470          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9471          */
9472         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9473
9474         if (val & LCPLL_POWER_DOWN_ALLOW) {
9475                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9476                 I915_WRITE(LCPLL_CTL, val);
9477                 POSTING_READ(LCPLL_CTL);
9478         }
9479
9480         val = hsw_read_dcomp(dev_priv);
9481         val |= D_COMP_COMP_FORCE;
9482         val &= ~D_COMP_COMP_DISABLE;
9483         hsw_write_dcomp(dev_priv, val);
9484
9485         val = I915_READ(LCPLL_CTL);
9486         val &= ~LCPLL_PLL_DISABLE;
9487         I915_WRITE(LCPLL_CTL, val);
9488
9489         if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5))
9490                 DRM_ERROR("LCPLL not locked yet\n");
9491
9492         if (val & LCPLL_CD_SOURCE_FCLK) {
9493                 val = I915_READ(LCPLL_CTL);
9494                 val &= ~LCPLL_CD_SOURCE_FCLK;
9495                 I915_WRITE(LCPLL_CTL, val);
9496
9497                 if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9498                                         LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9499                         DRM_ERROR("Switching back to LCPLL failed\n");
9500         }
9501
9502         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9503         intel_update_cdclk(dev_priv->dev);
9504 }
9505
9506 /*
9507  * Package states C8 and deeper are really deep PC states that can only be
9508  * reached when all the devices on the system allow it, so even if the graphics
9509  * device allows PC8+, it doesn't mean the system will actually get to these
9510  * states. Our driver only allows PC8+ when going into runtime PM.
9511  *
9512  * The requirements for PC8+ are that all the outputs are disabled, the power
9513  * well is disabled and most interrupts are disabled, and these are also
9514  * requirements for runtime PM. When these conditions are met, we manually do
9515  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9516  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9517  * hang the machine.
9518  *
9519  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9520  * the state of some registers, so when we come back from PC8+ we need to
9521  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9522  * need to take care of the registers kept by RC6. Notice that this happens even
9523  * if we don't put the device in PCI D3 state (which is what currently happens
9524  * because of the runtime PM support).
9525  *
9526  * For more, read "Display Sequences for Package C8" on the hardware
9527  * documentation.
9528  */
9529 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9530 {
9531         struct drm_device *dev = dev_priv->dev;
9532         uint32_t val;
9533
9534         DRM_DEBUG_KMS("Enabling package C8+\n");
9535
9536         if (HAS_PCH_LPT_LP(dev)) {
9537                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9538                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9539                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9540         }
9541
9542         lpt_disable_clkout_dp(dev);
9543         hsw_disable_lcpll(dev_priv, true, true);
9544 }
9545
9546 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9547 {
9548         struct drm_device *dev = dev_priv->dev;
9549         uint32_t val;
9550
9551         DRM_DEBUG_KMS("Disabling package C8+\n");
9552
9553         hsw_restore_lcpll(dev_priv);
9554         lpt_init_pch_refclk(dev);
9555
9556         if (HAS_PCH_LPT_LP(dev)) {
9557                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9558                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9559                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9560         }
9561
9562         intel_prepare_ddi(dev);
9563 }
9564
9565 static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9566 {
9567         struct drm_device *dev = old_state->dev;
9568         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9569
9570         broxton_set_cdclk(dev, req_cdclk);
9571 }
9572
9573 /* compute the max rate for new configuration */
9574 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
9575 {
9576         struct intel_crtc *intel_crtc;
9577         struct intel_crtc_state *crtc_state;
9578         int max_pixel_rate = 0;
9579
9580         for_each_intel_crtc(state->dev, intel_crtc) {
9581                 int pixel_rate;
9582
9583                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
9584                 if (IS_ERR(crtc_state))
9585                         return PTR_ERR(crtc_state);
9586
9587                 if (!crtc_state->base.enable)
9588                         continue;
9589
9590                 pixel_rate = ilk_pipe_pixel_rate(crtc_state);
9591
9592                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
9593                 if (IS_BROADWELL(state->dev) && crtc_state->ips_enabled)
9594                         pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95);
9595
9596                 max_pixel_rate = max(max_pixel_rate, pixel_rate);
9597         }
9598
9599         return max_pixel_rate;
9600 }
9601
9602 static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
9603 {
9604         struct drm_i915_private *dev_priv = dev->dev_private;
9605         uint32_t val, data;
9606         int ret;
9607
9608         if (WARN((I915_READ(LCPLL_CTL) &
9609                   (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
9610                    LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
9611                    LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
9612                    LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
9613                  "trying to change cdclk frequency with cdclk not enabled\n"))
9614                 return;
9615
9616         mutex_lock(&dev_priv->rps.hw_lock);
9617         ret = sandybridge_pcode_write(dev_priv,
9618                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
9619         mutex_unlock(&dev_priv->rps.hw_lock);
9620         if (ret) {
9621                 DRM_ERROR("failed to inform pcode about cdclk change\n");
9622                 return;
9623         }
9624
9625         val = I915_READ(LCPLL_CTL);
9626         val |= LCPLL_CD_SOURCE_FCLK;
9627         I915_WRITE(LCPLL_CTL, val);
9628
9629         if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9630                                LCPLL_CD_SOURCE_FCLK_DONE, 1))
9631                 DRM_ERROR("Switching to FCLK failed\n");
9632
9633         val = I915_READ(LCPLL_CTL);
9634         val &= ~LCPLL_CLK_FREQ_MASK;
9635
9636         switch (cdclk) {
9637         case 450000:
9638                 val |= LCPLL_CLK_FREQ_450;
9639                 data = 0;
9640                 break;
9641         case 540000:
9642                 val |= LCPLL_CLK_FREQ_54O_BDW;
9643                 data = 1;
9644                 break;
9645         case 337500:
9646                 val |= LCPLL_CLK_FREQ_337_5_BDW;
9647                 data = 2;
9648                 break;
9649         case 675000:
9650                 val |= LCPLL_CLK_FREQ_675_BDW;
9651                 data = 3;
9652                 break;
9653         default:
9654                 WARN(1, "invalid cdclk frequency\n");
9655                 return;
9656         }
9657
9658         I915_WRITE(LCPLL_CTL, val);
9659
9660         val = I915_READ(LCPLL_CTL);
9661         val &= ~LCPLL_CD_SOURCE_FCLK;
9662         I915_WRITE(LCPLL_CTL, val);
9663
9664         if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9665                                 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9666                 DRM_ERROR("Switching back to LCPLL failed\n");
9667
9668         mutex_lock(&dev_priv->rps.hw_lock);
9669         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
9670         mutex_unlock(&dev_priv->rps.hw_lock);
9671
9672         intel_update_cdclk(dev);
9673
9674         WARN(cdclk != dev_priv->cdclk_freq,
9675              "cdclk requested %d kHz but got %d kHz\n",
9676              cdclk, dev_priv->cdclk_freq);
9677 }
9678
9679 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
9680 {
9681         struct drm_i915_private *dev_priv = to_i915(state->dev);
9682         int max_pixclk = ilk_max_pixel_rate(state);
9683         int cdclk;
9684
9685         /*
9686          * FIXME should also account for plane ratio
9687          * once 64bpp pixel formats are supported.
9688          */
9689         if (max_pixclk > 540000)
9690                 cdclk = 675000;
9691         else if (max_pixclk > 450000)
9692                 cdclk = 540000;
9693         else if (max_pixclk > 337500)
9694                 cdclk = 450000;
9695         else
9696                 cdclk = 337500;
9697
9698         /*
9699          * FIXME move the cdclk caclulation to
9700          * compute_config() so we can fail gracegully.
9701          */
9702         if (cdclk > dev_priv->max_cdclk_freq) {
9703                 DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
9704                           cdclk, dev_priv->max_cdclk_freq);
9705                 cdclk = dev_priv->max_cdclk_freq;
9706         }
9707
9708         to_intel_atomic_state(state)->cdclk = cdclk;
9709
9710         return 0;
9711 }
9712
9713 static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9714 {
9715         struct drm_device *dev = old_state->dev;
9716         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9717
9718         broadwell_set_cdclk(dev, req_cdclk);
9719 }
9720
9721 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9722                                       struct intel_crtc_state *crtc_state)
9723 {
9724         if (!intel_ddi_pll_select(crtc, crtc_state))
9725                 return -EINVAL;
9726
9727         crtc->lowfreq_avail = false;
9728
9729         return 0;
9730 }
9731
9732 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9733                                 enum port port,
9734                                 struct intel_crtc_state *pipe_config)
9735 {
9736         switch (port) {
9737         case PORT_A:
9738                 pipe_config->ddi_pll_sel = SKL_DPLL0;
9739                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9740                 break;
9741         case PORT_B:
9742                 pipe_config->ddi_pll_sel = SKL_DPLL1;
9743                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9744                 break;
9745         case PORT_C:
9746                 pipe_config->ddi_pll_sel = SKL_DPLL2;
9747                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9748                 break;
9749         default:
9750                 DRM_ERROR("Incorrect port type\n");
9751         }
9752 }
9753
9754 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9755                                 enum port port,
9756                                 struct intel_crtc_state *pipe_config)
9757 {
9758         u32 temp, dpll_ctl1;
9759
9760         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9761         pipe_config->ddi_pll_sel = temp >> (port * 3 + 1);
9762
9763         switch (pipe_config->ddi_pll_sel) {
9764         case SKL_DPLL0:
9765                 /*
9766                  * On SKL the eDP DPLL (DPLL0 as we don't use SSC) is not part
9767                  * of the shared DPLL framework and thus needs to be read out
9768                  * separately
9769                  */
9770                 dpll_ctl1 = I915_READ(DPLL_CTRL1);
9771                 pipe_config->dpll_hw_state.ctrl1 = dpll_ctl1 & 0x3f;
9772                 break;
9773         case SKL_DPLL1:
9774                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9775                 break;
9776         case SKL_DPLL2:
9777                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9778                 break;
9779         case SKL_DPLL3:
9780                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9781                 break;
9782         }
9783 }
9784
9785 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9786                                 enum port port,
9787                                 struct intel_crtc_state *pipe_config)
9788 {
9789         pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9790
9791         switch (pipe_config->ddi_pll_sel) {
9792         case PORT_CLK_SEL_WRPLL1:
9793                 pipe_config->shared_dpll = DPLL_ID_WRPLL1;
9794                 break;
9795         case PORT_CLK_SEL_WRPLL2:
9796                 pipe_config->shared_dpll = DPLL_ID_WRPLL2;
9797                 break;
9798         case PORT_CLK_SEL_SPLL:
9799                 pipe_config->shared_dpll = DPLL_ID_SPLL;
9800         }
9801 }
9802
9803 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9804                                        struct intel_crtc_state *pipe_config)
9805 {
9806         struct drm_device *dev = crtc->base.dev;
9807         struct drm_i915_private *dev_priv = dev->dev_private;
9808         struct intel_shared_dpll *pll;
9809         enum port port;
9810         uint32_t tmp;
9811
9812         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9813
9814         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9815
9816         if (IS_SKYLAKE(dev))
9817                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9818         else if (IS_BROXTON(dev))
9819                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9820         else
9821                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9822
9823         if (pipe_config->shared_dpll >= 0) {
9824                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9825
9826                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9827                                            &pipe_config->dpll_hw_state));
9828         }
9829
9830         /*
9831          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9832          * DDI E. So just check whether this pipe is wired to DDI E and whether
9833          * the PCH transcoder is on.
9834          */
9835         if (INTEL_INFO(dev)->gen < 9 &&
9836             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9837                 pipe_config->has_pch_encoder = true;
9838
9839                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9840                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9841                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9842
9843                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9844         }
9845 }
9846
9847 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9848                                     struct intel_crtc_state *pipe_config)
9849 {
9850         struct drm_device *dev = crtc->base.dev;
9851         struct drm_i915_private *dev_priv = dev->dev_private;
9852         enum intel_display_power_domain pfit_domain;
9853         uint32_t tmp;
9854
9855         if (!intel_display_power_is_enabled(dev_priv,
9856                                          POWER_DOMAIN_PIPE(crtc->pipe)))
9857                 return false;
9858
9859         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9860         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9861
9862         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
9863         if (tmp & TRANS_DDI_FUNC_ENABLE) {
9864                 enum pipe trans_edp_pipe;
9865                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9866                 default:
9867                         WARN(1, "unknown pipe linked to edp transcoder\n");
9868                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9869                 case TRANS_DDI_EDP_INPUT_A_ON:
9870                         trans_edp_pipe = PIPE_A;
9871                         break;
9872                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9873                         trans_edp_pipe = PIPE_B;
9874                         break;
9875                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9876                         trans_edp_pipe = PIPE_C;
9877                         break;
9878                 }
9879
9880                 if (trans_edp_pipe == crtc->pipe)
9881                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
9882         }
9883
9884         if (!intel_display_power_is_enabled(dev_priv,
9885                         POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
9886                 return false;
9887
9888         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9889         if (!(tmp & PIPECONF_ENABLE))
9890                 return false;
9891
9892         haswell_get_ddi_port_state(crtc, pipe_config);
9893
9894         intel_get_pipe_timings(crtc, pipe_config);
9895
9896         if (INTEL_INFO(dev)->gen >= 9) {
9897                 skl_init_scalers(dev, crtc, pipe_config);
9898         }
9899
9900         pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9901
9902         if (INTEL_INFO(dev)->gen >= 9) {
9903                 pipe_config->scaler_state.scaler_id = -1;
9904                 pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
9905         }
9906
9907         if (intel_display_power_is_enabled(dev_priv, pfit_domain)) {
9908                 if (INTEL_INFO(dev)->gen >= 9)
9909                         skylake_get_pfit_config(crtc, pipe_config);
9910                 else
9911                         ironlake_get_pfit_config(crtc, pipe_config);
9912         }
9913
9914         if (IS_HASWELL(dev))
9915                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
9916                         (I915_READ(IPS_CTL) & IPS_ENABLE);
9917
9918         if (pipe_config->cpu_transcoder != TRANSCODER_EDP) {
9919                 pipe_config->pixel_multiplier =
9920                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9921         } else {
9922                 pipe_config->pixel_multiplier = 1;
9923         }
9924
9925         return true;
9926 }
9927
9928 static void i845_update_cursor(struct drm_crtc *crtc, u32 base, bool on)
9929 {
9930         struct drm_device *dev = crtc->dev;
9931         struct drm_i915_private *dev_priv = dev->dev_private;
9932         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9933         uint32_t cntl = 0, size = 0;
9934
9935         if (on) {
9936                 unsigned int width = intel_crtc->base.cursor->state->crtc_w;
9937                 unsigned int height = intel_crtc->base.cursor->state->crtc_h;
9938                 unsigned int stride = roundup_pow_of_two(width) * 4;
9939
9940                 switch (stride) {
9941                 default:
9942                         WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
9943                                   width, stride);
9944                         stride = 256;
9945                         /* fallthrough */
9946                 case 256:
9947                 case 512:
9948                 case 1024:
9949                 case 2048:
9950                         break;
9951                 }
9952
9953                 cntl |= CURSOR_ENABLE |
9954                         CURSOR_GAMMA_ENABLE |
9955                         CURSOR_FORMAT_ARGB |
9956                         CURSOR_STRIDE(stride);
9957
9958                 size = (height << 12) | width;
9959         }
9960
9961         if (intel_crtc->cursor_cntl != 0 &&
9962             (intel_crtc->cursor_base != base ||
9963              intel_crtc->cursor_size != size ||
9964              intel_crtc->cursor_cntl != cntl)) {
9965                 /* On these chipsets we can only modify the base/size/stride
9966                  * whilst the cursor is disabled.
9967                  */
9968                 I915_WRITE(CURCNTR(PIPE_A), 0);
9969                 POSTING_READ(CURCNTR(PIPE_A));
9970                 intel_crtc->cursor_cntl = 0;
9971         }
9972
9973         if (intel_crtc->cursor_base != base) {
9974                 I915_WRITE(CURBASE(PIPE_A), base);
9975                 intel_crtc->cursor_base = base;
9976         }
9977
9978         if (intel_crtc->cursor_size != size) {
9979                 I915_WRITE(CURSIZE, size);
9980                 intel_crtc->cursor_size = size;
9981         }
9982
9983         if (intel_crtc->cursor_cntl != cntl) {
9984                 I915_WRITE(CURCNTR(PIPE_A), cntl);
9985                 POSTING_READ(CURCNTR(PIPE_A));
9986                 intel_crtc->cursor_cntl = cntl;
9987         }
9988 }
9989
9990 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base, bool on)
9991 {
9992         struct drm_device *dev = crtc->dev;
9993         struct drm_i915_private *dev_priv = dev->dev_private;
9994         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9995         int pipe = intel_crtc->pipe;
9996         uint32_t cntl = 0;
9997
9998         if (on) {
9999                 cntl = MCURSOR_GAMMA_ENABLE;
10000                 switch (intel_crtc->base.cursor->state->crtc_w) {
10001                         case 64:
10002                                 cntl |= CURSOR_MODE_64_ARGB_AX;
10003                                 break;
10004                         case 128:
10005                                 cntl |= CURSOR_MODE_128_ARGB_AX;
10006                                 break;
10007                         case 256:
10008                                 cntl |= CURSOR_MODE_256_ARGB_AX;
10009                                 break;
10010                         default:
10011                                 MISSING_CASE(intel_crtc->base.cursor->state->crtc_w);
10012                                 return;
10013                 }
10014                 cntl |= pipe << 28; /* Connect to correct pipe */
10015
10016                 if (HAS_DDI(dev))
10017                         cntl |= CURSOR_PIPE_CSC_ENABLE;
10018         }
10019
10020         if (crtc->cursor->state->rotation == BIT(DRM_ROTATE_180))
10021                 cntl |= CURSOR_ROTATE_180;
10022
10023         if (intel_crtc->cursor_cntl != cntl) {
10024                 I915_WRITE(CURCNTR(pipe), cntl);
10025                 POSTING_READ(CURCNTR(pipe));
10026                 intel_crtc->cursor_cntl = cntl;
10027         }
10028
10029         /* and commit changes on next vblank */
10030         I915_WRITE(CURBASE(pipe), base);
10031         POSTING_READ(CURBASE(pipe));
10032
10033         intel_crtc->cursor_base = base;
10034 }
10035
10036 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
10037 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
10038                                      bool on)
10039 {
10040         struct drm_device *dev = crtc->dev;
10041         struct drm_i915_private *dev_priv = dev->dev_private;
10042         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10043         int pipe = intel_crtc->pipe;
10044         struct drm_plane_state *cursor_state = crtc->cursor->state;
10045         int x = cursor_state->crtc_x;
10046         int y = cursor_state->crtc_y;
10047         u32 base = 0, pos = 0;
10048
10049         base = intel_crtc->cursor_addr;
10050
10051         if (x >= intel_crtc->config->pipe_src_w)
10052                 on = false;
10053
10054         if (y >= intel_crtc->config->pipe_src_h)
10055                 on = false;
10056
10057         if (x < 0) {
10058                 if (x + cursor_state->crtc_w <= 0)
10059                         on = false;
10060
10061                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10062                 x = -x;
10063         }
10064         pos |= x << CURSOR_X_SHIFT;
10065
10066         if (y < 0) {
10067                 if (y + cursor_state->crtc_h <= 0)
10068                         on = false;
10069
10070                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10071                 y = -y;
10072         }
10073         pos |= y << CURSOR_Y_SHIFT;
10074
10075         I915_WRITE(CURPOS(pipe), pos);
10076
10077         /* ILK+ do this automagically */
10078         if (HAS_GMCH_DISPLAY(dev) &&
10079             crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) {
10080                 base += (cursor_state->crtc_h *
10081                          cursor_state->crtc_w - 1) * 4;
10082         }
10083
10084         if (IS_845G(dev) || IS_I865G(dev))
10085                 i845_update_cursor(crtc, base, on);
10086         else
10087                 i9xx_update_cursor(crtc, base, on);
10088 }
10089
10090 static bool cursor_size_ok(struct drm_device *dev,
10091                            uint32_t width, uint32_t height)
10092 {
10093         if (width == 0 || height == 0)
10094                 return false;
10095
10096         /*
10097          * 845g/865g are special in that they are only limited by
10098          * the width of their cursors, the height is arbitrary up to
10099          * the precision of the register. Everything else requires
10100          * square cursors, limited to a few power-of-two sizes.
10101          */
10102         if (IS_845G(dev) || IS_I865G(dev)) {
10103                 if ((width & 63) != 0)
10104                         return false;
10105
10106                 if (width > (IS_845G(dev) ? 64 : 512))
10107                         return false;
10108
10109                 if (height > 1023)
10110                         return false;
10111         } else {
10112                 switch (width | height) {
10113                 case 256:
10114                 case 128:
10115                         if (IS_GEN2(dev))
10116                                 return false;
10117                 case 64:
10118                         break;
10119                 default:
10120                         return false;
10121                 }
10122         }
10123
10124         return true;
10125 }
10126
10127 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
10128                                  u16 *blue, uint32_t start, uint32_t size)
10129 {
10130         int end = (start + size > 256) ? 256 : start + size, i;
10131         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10132
10133         for (i = start; i < end; i++) {
10134                 intel_crtc->lut_r[i] = red[i] >> 8;
10135                 intel_crtc->lut_g[i] = green[i] >> 8;
10136                 intel_crtc->lut_b[i] = blue[i] >> 8;
10137         }
10138
10139         intel_crtc_load_lut(crtc);
10140 }
10141
10142 /* VESA 640x480x72Hz mode to set on the pipe */
10143 static struct drm_display_mode load_detect_mode = {
10144         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10145                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10146 };
10147
10148 struct drm_framebuffer *
10149 __intel_framebuffer_create(struct drm_device *dev,
10150                            struct drm_mode_fb_cmd2 *mode_cmd,
10151                            struct drm_i915_gem_object *obj)
10152 {
10153         struct intel_framebuffer *intel_fb;
10154         int ret;
10155
10156         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10157         if (!intel_fb) {
10158                 drm_gem_object_unreference(&obj->base);
10159                 return ERR_PTR(-ENOMEM);
10160         }
10161
10162         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
10163         if (ret)
10164                 goto err;
10165
10166         return &intel_fb->base;
10167 err:
10168         drm_gem_object_unreference(&obj->base);
10169         kfree(intel_fb);
10170
10171         return ERR_PTR(ret);
10172 }
10173
10174 static struct drm_framebuffer *
10175 intel_framebuffer_create(struct drm_device *dev,
10176                          struct drm_mode_fb_cmd2 *mode_cmd,
10177                          struct drm_i915_gem_object *obj)
10178 {
10179         struct drm_framebuffer *fb;
10180         int ret;
10181
10182         ret = i915_mutex_lock_interruptible(dev);
10183         if (ret)
10184                 return ERR_PTR(ret);
10185         fb = __intel_framebuffer_create(dev, mode_cmd, obj);
10186         mutex_unlock(&dev->struct_mutex);
10187
10188         return fb;
10189 }
10190
10191 static u32
10192 intel_framebuffer_pitch_for_width(int width, int bpp)
10193 {
10194         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
10195         return ALIGN(pitch, 64);
10196 }
10197
10198 static u32
10199 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
10200 {
10201         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
10202         return PAGE_ALIGN(pitch * mode->vdisplay);
10203 }
10204
10205 static struct drm_framebuffer *
10206 intel_framebuffer_create_for_mode(struct drm_device *dev,
10207                                   struct drm_display_mode *mode,
10208                                   int depth, int bpp)
10209 {
10210         struct drm_i915_gem_object *obj;
10211         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
10212
10213         obj = i915_gem_alloc_object(dev,
10214                                     intel_framebuffer_size_for_mode(mode, bpp));
10215         if (obj == NULL)
10216                 return ERR_PTR(-ENOMEM);
10217
10218         mode_cmd.width = mode->hdisplay;
10219         mode_cmd.height = mode->vdisplay;
10220         mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
10221                                                                 bpp);
10222         mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
10223
10224         return intel_framebuffer_create(dev, &mode_cmd, obj);
10225 }
10226
10227 static struct drm_framebuffer *
10228 mode_fits_in_fbdev(struct drm_device *dev,
10229                    struct drm_display_mode *mode)
10230 {
10231 #ifdef CONFIG_DRM_FBDEV_EMULATION
10232         struct drm_i915_private *dev_priv = dev->dev_private;
10233         struct drm_i915_gem_object *obj;
10234         struct drm_framebuffer *fb;
10235
10236         if (!dev_priv->fbdev)
10237                 return NULL;
10238
10239         if (!dev_priv->fbdev->fb)
10240                 return NULL;
10241
10242         obj = dev_priv->fbdev->fb->obj;
10243         BUG_ON(!obj);
10244
10245         fb = &dev_priv->fbdev->fb->base;
10246         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
10247                                                                fb->bits_per_pixel))
10248                 return NULL;
10249
10250         if (obj->base.size < mode->vdisplay * fb->pitches[0])
10251                 return NULL;
10252
10253         return fb;
10254 #else
10255         return NULL;
10256 #endif
10257 }
10258
10259 static int intel_modeset_setup_plane_state(struct drm_atomic_state *state,
10260                                            struct drm_crtc *crtc,
10261                                            struct drm_display_mode *mode,
10262                                            struct drm_framebuffer *fb,
10263                                            int x, int y)
10264 {
10265         struct drm_plane_state *plane_state;
10266         int hdisplay, vdisplay;
10267         int ret;
10268
10269         plane_state = drm_atomic_get_plane_state(state, crtc->primary);
10270         if (IS_ERR(plane_state))
10271                 return PTR_ERR(plane_state);
10272
10273         if (mode)
10274                 drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
10275         else
10276                 hdisplay = vdisplay = 0;
10277
10278         ret = drm_atomic_set_crtc_for_plane(plane_state, fb ? crtc : NULL);
10279         if (ret)
10280                 return ret;
10281         drm_atomic_set_fb_for_plane(plane_state, fb);
10282         plane_state->crtc_x = 0;
10283         plane_state->crtc_y = 0;
10284         plane_state->crtc_w = hdisplay;
10285         plane_state->crtc_h = vdisplay;
10286         plane_state->src_x = x << 16;
10287         plane_state->src_y = y << 16;
10288         plane_state->src_w = hdisplay << 16;
10289         plane_state->src_h = vdisplay << 16;
10290
10291         return 0;
10292 }
10293
10294 bool intel_get_load_detect_pipe(struct drm_connector *connector,
10295                                 struct drm_display_mode *mode,
10296                                 struct intel_load_detect_pipe *old,
10297                                 struct drm_modeset_acquire_ctx *ctx)
10298 {
10299         struct intel_crtc *intel_crtc;
10300         struct intel_encoder *intel_encoder =
10301                 intel_attached_encoder(connector);
10302         struct drm_crtc *possible_crtc;
10303         struct drm_encoder *encoder = &intel_encoder->base;
10304         struct drm_crtc *crtc = NULL;
10305         struct drm_device *dev = encoder->dev;
10306         struct drm_framebuffer *fb;
10307         struct drm_mode_config *config = &dev->mode_config;
10308         struct drm_atomic_state *state = NULL;
10309         struct drm_connector_state *connector_state;
10310         struct intel_crtc_state *crtc_state;
10311         int ret, i = -1;
10312
10313         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10314                       connector->base.id, connector->name,
10315                       encoder->base.id, encoder->name);
10316
10317 retry:
10318         ret = drm_modeset_lock(&config->connection_mutex, ctx);
10319         if (ret)
10320                 goto fail;
10321
10322         /*
10323          * Algorithm gets a little messy:
10324          *
10325          *   - if the connector already has an assigned crtc, use it (but make
10326          *     sure it's on first)
10327          *
10328          *   - try to find the first unused crtc that can drive this connector,
10329          *     and use that if we find one
10330          */
10331
10332         /* See if we already have a CRTC for this connector */
10333         if (encoder->crtc) {
10334                 crtc = encoder->crtc;
10335
10336                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10337                 if (ret)
10338                         goto fail;
10339                 ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10340                 if (ret)
10341                         goto fail;
10342
10343                 old->dpms_mode = connector->dpms;
10344                 old->load_detect_temp = false;
10345
10346                 /* Make sure the crtc and connector are running */
10347                 if (connector->dpms != DRM_MODE_DPMS_ON)
10348                         connector->funcs->dpms(connector, DRM_MODE_DPMS_ON);
10349
10350                 return true;
10351         }
10352
10353         /* Find an unused one (if possible) */
10354         for_each_crtc(dev, possible_crtc) {
10355                 i++;
10356                 if (!(encoder->possible_crtcs & (1 << i)))
10357                         continue;
10358                 if (possible_crtc->state->enable)
10359                         continue;
10360
10361                 crtc = possible_crtc;
10362                 break;
10363         }
10364
10365         /*
10366          * If we didn't find an unused CRTC, don't use any.
10367          */
10368         if (!crtc) {
10369                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10370                 goto fail;
10371         }
10372
10373         ret = drm_modeset_lock(&crtc->mutex, ctx);
10374         if (ret)
10375                 goto fail;
10376         ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10377         if (ret)
10378                 goto fail;
10379
10380         intel_crtc = to_intel_crtc(crtc);
10381         old->dpms_mode = connector->dpms;
10382         old->load_detect_temp = true;
10383         old->release_fb = NULL;
10384
10385         state = drm_atomic_state_alloc(dev);
10386         if (!state)
10387                 return false;
10388
10389         state->acquire_ctx = ctx;
10390
10391         connector_state = drm_atomic_get_connector_state(state, connector);
10392         if (IS_ERR(connector_state)) {
10393                 ret = PTR_ERR(connector_state);
10394                 goto fail;
10395         }
10396
10397         connector_state->crtc = crtc;
10398         connector_state->best_encoder = &intel_encoder->base;
10399
10400         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10401         if (IS_ERR(crtc_state)) {
10402                 ret = PTR_ERR(crtc_state);
10403                 goto fail;
10404         }
10405
10406         crtc_state->base.active = crtc_state->base.enable = true;
10407
10408         if (!mode)
10409                 mode = &load_detect_mode;
10410
10411         /* We need a framebuffer large enough to accommodate all accesses
10412          * that the plane may generate whilst we perform load detection.
10413          * We can not rely on the fbcon either being present (we get called
10414          * during its initialisation to detect all boot displays, or it may
10415          * not even exist) or that it is large enough to satisfy the
10416          * requested mode.
10417          */
10418         fb = mode_fits_in_fbdev(dev, mode);
10419         if (fb == NULL) {
10420                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
10421                 fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
10422                 old->release_fb = fb;
10423         } else
10424                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
10425         if (IS_ERR(fb)) {
10426                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
10427                 goto fail;
10428         }
10429
10430         ret = intel_modeset_setup_plane_state(state, crtc, mode, fb, 0, 0);
10431         if (ret)
10432                 goto fail;
10433
10434         drm_mode_copy(&crtc_state->base.mode, mode);
10435
10436         if (drm_atomic_commit(state)) {
10437                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10438                 if (old->release_fb)
10439                         old->release_fb->funcs->destroy(old->release_fb);
10440                 goto fail;
10441         }
10442         crtc->primary->crtc = crtc;
10443
10444         /* let the connector get through one full cycle before testing */
10445         intel_wait_for_vblank(dev, intel_crtc->pipe);
10446         return true;
10447
10448 fail:
10449         drm_atomic_state_free(state);
10450         state = NULL;
10451
10452         if (ret == -EDEADLK) {
10453                 drm_modeset_backoff(ctx);
10454                 goto retry;
10455         }
10456
10457         return false;
10458 }
10459
10460 void intel_release_load_detect_pipe(struct drm_connector *connector,
10461                                     struct intel_load_detect_pipe *old,
10462                                     struct drm_modeset_acquire_ctx *ctx)
10463 {
10464         struct drm_device *dev = connector->dev;
10465         struct intel_encoder *intel_encoder =
10466                 intel_attached_encoder(connector);
10467         struct drm_encoder *encoder = &intel_encoder->base;
10468         struct drm_crtc *crtc = encoder->crtc;
10469         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10470         struct drm_atomic_state *state;
10471         struct drm_connector_state *connector_state;
10472         struct intel_crtc_state *crtc_state;
10473         int ret;
10474
10475         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10476                       connector->base.id, connector->name,
10477                       encoder->base.id, encoder->name);
10478
10479         if (old->load_detect_temp) {
10480                 state = drm_atomic_state_alloc(dev);
10481                 if (!state)
10482                         goto fail;
10483
10484                 state->acquire_ctx = ctx;
10485
10486                 connector_state = drm_atomic_get_connector_state(state, connector);
10487                 if (IS_ERR(connector_state))
10488                         goto fail;
10489
10490                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10491                 if (IS_ERR(crtc_state))
10492                         goto fail;
10493
10494                 connector_state->best_encoder = NULL;
10495                 connector_state->crtc = NULL;
10496
10497                 crtc_state->base.enable = crtc_state->base.active = false;
10498
10499                 ret = intel_modeset_setup_plane_state(state, crtc, NULL, NULL,
10500                                                       0, 0);
10501                 if (ret)
10502                         goto fail;
10503
10504                 ret = drm_atomic_commit(state);
10505                 if (ret)
10506                         goto fail;
10507
10508                 if (old->release_fb) {
10509                         drm_framebuffer_unregister_private(old->release_fb);
10510                         drm_framebuffer_unreference(old->release_fb);
10511                 }
10512
10513                 return;
10514         }
10515
10516         /* Switch crtc and encoder back off if necessary */
10517         if (old->dpms_mode != DRM_MODE_DPMS_ON)
10518                 connector->funcs->dpms(connector, old->dpms_mode);
10519
10520         return;
10521 fail:
10522         DRM_DEBUG_KMS("Couldn't release load detect pipe.\n");
10523         drm_atomic_state_free(state);
10524 }
10525
10526 static int i9xx_pll_refclk(struct drm_device *dev,
10527                            const struct intel_crtc_state *pipe_config)
10528 {
10529         struct drm_i915_private *dev_priv = dev->dev_private;
10530         u32 dpll = pipe_config->dpll_hw_state.dpll;
10531
10532         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10533                 return dev_priv->vbt.lvds_ssc_freq;
10534         else if (HAS_PCH_SPLIT(dev))
10535                 return 120000;
10536         else if (!IS_GEN2(dev))
10537                 return 96000;
10538         else
10539                 return 48000;
10540 }
10541
10542 /* Returns the clock of the currently programmed mode of the given pipe. */
10543 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10544                                 struct intel_crtc_state *pipe_config)
10545 {
10546         struct drm_device *dev = crtc->base.dev;
10547         struct drm_i915_private *dev_priv = dev->dev_private;
10548         int pipe = pipe_config->cpu_transcoder;
10549         u32 dpll = pipe_config->dpll_hw_state.dpll;
10550         u32 fp;
10551         intel_clock_t clock;
10552         int port_clock;
10553         int refclk = i9xx_pll_refclk(dev, pipe_config);
10554
10555         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10556                 fp = pipe_config->dpll_hw_state.fp0;
10557         else
10558                 fp = pipe_config->dpll_hw_state.fp1;
10559
10560         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10561         if (IS_PINEVIEW(dev)) {
10562                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10563                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10564         } else {
10565                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10566                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10567         }
10568
10569         if (!IS_GEN2(dev)) {
10570                 if (IS_PINEVIEW(dev))
10571                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10572                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10573                 else
10574                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10575                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10576
10577                 switch (dpll & DPLL_MODE_MASK) {
10578                 case DPLLB_MODE_DAC_SERIAL:
10579                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10580                                 5 : 10;
10581                         break;
10582                 case DPLLB_MODE_LVDS:
10583                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10584                                 7 : 14;
10585                         break;
10586                 default:
10587                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10588                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10589                         return;
10590                 }
10591
10592                 if (IS_PINEVIEW(dev))
10593                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10594                 else
10595                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10596         } else {
10597                 u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
10598                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10599
10600                 if (is_lvds) {
10601                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10602                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10603
10604                         if (lvds & LVDS_CLKB_POWER_UP)
10605                                 clock.p2 = 7;
10606                         else
10607                                 clock.p2 = 14;
10608                 } else {
10609                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10610                                 clock.p1 = 2;
10611                         else {
10612                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10613                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10614                         }
10615                         if (dpll & PLL_P2_DIVIDE_BY_4)
10616                                 clock.p2 = 4;
10617                         else
10618                                 clock.p2 = 2;
10619                 }
10620
10621                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10622         }
10623
10624         /*
10625          * This value includes pixel_multiplier. We will use
10626          * port_clock to compute adjusted_mode.crtc_clock in the
10627          * encoder's get_config() function.
10628          */
10629         pipe_config->port_clock = port_clock;
10630 }
10631
10632 int intel_dotclock_calculate(int link_freq,
10633                              const struct intel_link_m_n *m_n)
10634 {
10635         /*
10636          * The calculation for the data clock is:
10637          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10638          * But we want to avoid losing precison if possible, so:
10639          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10640          *
10641          * and the link clock is simpler:
10642          * link_clock = (m * link_clock) / n
10643          */
10644
10645         if (!m_n->link_n)
10646                 return 0;
10647
10648         return div_u64((u64)m_n->link_m * link_freq, m_n->link_n);
10649 }
10650
10651 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10652                                    struct intel_crtc_state *pipe_config)
10653 {
10654         struct drm_device *dev = crtc->base.dev;
10655
10656         /* read out port_clock from the DPLL */
10657         i9xx_crtc_clock_get(crtc, pipe_config);
10658
10659         /*
10660          * This value does not include pixel_multiplier.
10661          * We will check that port_clock and adjusted_mode.crtc_clock
10662          * agree once we know their relationship in the encoder's
10663          * get_config() function.
10664          */
10665         pipe_config->base.adjusted_mode.crtc_clock =
10666                 intel_dotclock_calculate(intel_fdi_link_freq(dev) * 10000,
10667                                          &pipe_config->fdi_m_n);
10668 }
10669
10670 /** Returns the currently programmed mode of the given pipe. */
10671 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
10672                                              struct drm_crtc *crtc)
10673 {
10674         struct drm_i915_private *dev_priv = dev->dev_private;
10675         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10676         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
10677         struct drm_display_mode *mode;
10678         struct intel_crtc_state pipe_config;
10679         int htot = I915_READ(HTOTAL(cpu_transcoder));
10680         int hsync = I915_READ(HSYNC(cpu_transcoder));
10681         int vtot = I915_READ(VTOTAL(cpu_transcoder));
10682         int vsync = I915_READ(VSYNC(cpu_transcoder));
10683         enum pipe pipe = intel_crtc->pipe;
10684
10685         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10686         if (!mode)
10687                 return NULL;
10688
10689         /*
10690          * Construct a pipe_config sufficient for getting the clock info
10691          * back out of crtc_clock_get.
10692          *
10693          * Note, if LVDS ever uses a non-1 pixel multiplier, we'll need
10694          * to use a real value here instead.
10695          */
10696         pipe_config.cpu_transcoder = (enum transcoder) pipe;
10697         pipe_config.pixel_multiplier = 1;
10698         pipe_config.dpll_hw_state.dpll = I915_READ(DPLL(pipe));
10699         pipe_config.dpll_hw_state.fp0 = I915_READ(FP0(pipe));
10700         pipe_config.dpll_hw_state.fp1 = I915_READ(FP1(pipe));
10701         i9xx_crtc_clock_get(intel_crtc, &pipe_config);
10702
10703         mode->clock = pipe_config.port_clock / pipe_config.pixel_multiplier;
10704         mode->hdisplay = (htot & 0xffff) + 1;
10705         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
10706         mode->hsync_start = (hsync & 0xffff) + 1;
10707         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
10708         mode->vdisplay = (vtot & 0xffff) + 1;
10709         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
10710         mode->vsync_start = (vsync & 0xffff) + 1;
10711         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
10712
10713         drm_mode_set_name(mode);
10714
10715         return mode;
10716 }
10717
10718 void intel_mark_busy(struct drm_device *dev)
10719 {
10720         struct drm_i915_private *dev_priv = dev->dev_private;
10721
10722         if (dev_priv->mm.busy)
10723                 return;
10724
10725         intel_runtime_pm_get(dev_priv);
10726         i915_update_gfx_val(dev_priv);
10727         if (INTEL_INFO(dev)->gen >= 6)
10728                 gen6_rps_busy(dev_priv);
10729         dev_priv->mm.busy = true;
10730 }
10731
10732 void intel_mark_idle(struct drm_device *dev)
10733 {
10734         struct drm_i915_private *dev_priv = dev->dev_private;
10735
10736         if (!dev_priv->mm.busy)
10737                 return;
10738
10739         dev_priv->mm.busy = false;
10740
10741         if (INTEL_INFO(dev)->gen >= 6)
10742                 gen6_rps_idle(dev->dev_private);
10743
10744         intel_runtime_pm_put(dev_priv);
10745 }
10746
10747 static void intel_crtc_destroy(struct drm_crtc *crtc)
10748 {
10749         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10750         struct drm_device *dev = crtc->dev;
10751         struct intel_unpin_work *work;
10752
10753         spin_lock_irq(&dev->event_lock);
10754         work = intel_crtc->unpin_work;
10755         intel_crtc->unpin_work = NULL;
10756         spin_unlock_irq(&dev->event_lock);
10757
10758         if (work) {
10759                 cancel_work_sync(&work->work);
10760                 kfree(work);
10761         }
10762
10763         drm_crtc_cleanup(crtc);
10764
10765         kfree(intel_crtc);
10766 }
10767
10768 static void intel_unpin_work_fn(struct work_struct *__work)
10769 {
10770         struct intel_unpin_work *work =
10771                 container_of(__work, struct intel_unpin_work, work);
10772         struct intel_crtc *crtc = to_intel_crtc(work->crtc);
10773         struct drm_device *dev = crtc->base.dev;
10774         struct drm_plane *primary = crtc->base.primary;
10775
10776         mutex_lock(&dev->struct_mutex);
10777         intel_unpin_fb_obj(work->old_fb, primary->state);
10778         drm_gem_object_unreference(&work->pending_flip_obj->base);
10779
10780         if (work->flip_queued_req)
10781                 i915_gem_request_assign(&work->flip_queued_req, NULL);
10782         mutex_unlock(&dev->struct_mutex);
10783
10784         intel_frontbuffer_flip_complete(dev, to_intel_plane(primary)->frontbuffer_bit);
10785         drm_framebuffer_unreference(work->old_fb);
10786
10787         BUG_ON(atomic_read(&crtc->unpin_work_count) == 0);
10788         atomic_dec(&crtc->unpin_work_count);
10789
10790         kfree(work);
10791 }
10792
10793 static void do_intel_finish_page_flip(struct drm_device *dev,
10794                                       struct drm_crtc *crtc)
10795 {
10796         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10797         struct intel_unpin_work *work;
10798         unsigned long flags;
10799
10800         /* Ignore early vblank irqs */
10801         if (intel_crtc == NULL)
10802                 return;
10803
10804         /*
10805          * This is called both by irq handlers and the reset code (to complete
10806          * lost pageflips) so needs the full irqsave spinlocks.
10807          */
10808         spin_lock_irqsave(&dev->event_lock, flags);
10809         work = intel_crtc->unpin_work;
10810
10811         /* Ensure we don't miss a work->pending update ... */
10812         smp_rmb();
10813
10814         if (work == NULL || atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
10815                 spin_unlock_irqrestore(&dev->event_lock, flags);
10816                 return;
10817         }
10818
10819         page_flip_completed(intel_crtc);
10820
10821         spin_unlock_irqrestore(&dev->event_lock, flags);
10822 }
10823
10824 void intel_finish_page_flip(struct drm_device *dev, int pipe)
10825 {
10826         struct drm_i915_private *dev_priv = dev->dev_private;
10827         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
10828
10829         do_intel_finish_page_flip(dev, crtc);
10830 }
10831
10832 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
10833 {
10834         struct drm_i915_private *dev_priv = dev->dev_private;
10835         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
10836
10837         do_intel_finish_page_flip(dev, crtc);
10838 }
10839
10840 /* Is 'a' after or equal to 'b'? */
10841 static bool g4x_flip_count_after_eq(u32 a, u32 b)
10842 {
10843         return !((a - b) & 0x80000000);
10844 }
10845
10846 static bool page_flip_finished(struct intel_crtc *crtc)
10847 {
10848         struct drm_device *dev = crtc->base.dev;
10849         struct drm_i915_private *dev_priv = dev->dev_private;
10850
10851         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
10852             crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
10853                 return true;
10854
10855         /*
10856          * The relevant registers doen't exist on pre-ctg.
10857          * As the flip done interrupt doesn't trigger for mmio
10858          * flips on gmch platforms, a flip count check isn't
10859          * really needed there. But since ctg has the registers,
10860          * include it in the check anyway.
10861          */
10862         if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
10863                 return true;
10864
10865         /*
10866          * A DSPSURFLIVE check isn't enough in case the mmio and CS flips
10867          * used the same base address. In that case the mmio flip might
10868          * have completed, but the CS hasn't even executed the flip yet.
10869          *
10870          * A flip count check isn't enough as the CS might have updated
10871          * the base address just after start of vblank, but before we
10872          * managed to process the interrupt. This means we'd complete the
10873          * CS flip too soon.
10874          *
10875          * Combining both checks should get us a good enough result. It may
10876          * still happen that the CS flip has been executed, but has not
10877          * yet actually completed. But in case the base address is the same
10878          * anyway, we don't really care.
10879          */
10880         return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
10881                 crtc->unpin_work->gtt_offset &&
10882                 g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
10883                                     crtc->unpin_work->flip_count);
10884 }
10885
10886 void intel_prepare_page_flip(struct drm_device *dev, int plane)
10887 {
10888         struct drm_i915_private *dev_priv = dev->dev_private;
10889         struct intel_crtc *intel_crtc =
10890                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
10891         unsigned long flags;
10892
10893
10894         /*
10895          * This is called both by irq handlers and the reset code (to complete
10896          * lost pageflips) so needs the full irqsave spinlocks.
10897          *
10898          * NB: An MMIO update of the plane base pointer will also
10899          * generate a page-flip completion irq, i.e. every modeset
10900          * is also accompanied by a spurious intel_prepare_page_flip().
10901          */
10902         spin_lock_irqsave(&dev->event_lock, flags);
10903         if (intel_crtc->unpin_work && page_flip_finished(intel_crtc))
10904                 atomic_inc_not_zero(&intel_crtc->unpin_work->pending);
10905         spin_unlock_irqrestore(&dev->event_lock, flags);
10906 }
10907
10908 static inline void intel_mark_page_flip_active(struct intel_unpin_work *work)
10909 {
10910         /* Ensure that the work item is consistent when activating it ... */
10911         smp_wmb();
10912         atomic_set(&work->pending, INTEL_FLIP_PENDING);
10913         /* and that it is marked active as soon as the irq could fire. */
10914         smp_wmb();
10915 }
10916
10917 static int intel_gen2_queue_flip(struct drm_device *dev,
10918                                  struct drm_crtc *crtc,
10919                                  struct drm_framebuffer *fb,
10920                                  struct drm_i915_gem_object *obj,
10921                                  struct drm_i915_gem_request *req,
10922                                  uint32_t flags)
10923 {
10924         struct intel_engine_cs *ring = req->ring;
10925         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10926         u32 flip_mask;
10927         int ret;
10928
10929         ret = intel_ring_begin(req, 6);
10930         if (ret)
10931                 return ret;
10932
10933         /* Can't queue multiple flips, so wait for the previous
10934          * one to finish before executing the next.
10935          */
10936         if (intel_crtc->plane)
10937                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10938         else
10939                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10940         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10941         intel_ring_emit(ring, MI_NOOP);
10942         intel_ring_emit(ring, MI_DISPLAY_FLIP |
10943                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10944         intel_ring_emit(ring, fb->pitches[0]);
10945         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10946         intel_ring_emit(ring, 0); /* aux display base address, unused */
10947
10948         intel_mark_page_flip_active(intel_crtc->unpin_work);
10949         return 0;
10950 }
10951
10952 static int intel_gen3_queue_flip(struct drm_device *dev,
10953                                  struct drm_crtc *crtc,
10954                                  struct drm_framebuffer *fb,
10955                                  struct drm_i915_gem_object *obj,
10956                                  struct drm_i915_gem_request *req,
10957                                  uint32_t flags)
10958 {
10959         struct intel_engine_cs *ring = req->ring;
10960         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10961         u32 flip_mask;
10962         int ret;
10963
10964         ret = intel_ring_begin(req, 6);
10965         if (ret)
10966                 return ret;
10967
10968         if (intel_crtc->plane)
10969                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10970         else
10971                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10972         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10973         intel_ring_emit(ring, MI_NOOP);
10974         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
10975                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10976         intel_ring_emit(ring, fb->pitches[0]);
10977         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10978         intel_ring_emit(ring, MI_NOOP);
10979
10980         intel_mark_page_flip_active(intel_crtc->unpin_work);
10981         return 0;
10982 }
10983
10984 static int intel_gen4_queue_flip(struct drm_device *dev,
10985                                  struct drm_crtc *crtc,
10986                                  struct drm_framebuffer *fb,
10987                                  struct drm_i915_gem_object *obj,
10988                                  struct drm_i915_gem_request *req,
10989                                  uint32_t flags)
10990 {
10991         struct intel_engine_cs *ring = req->ring;
10992         struct drm_i915_private *dev_priv = dev->dev_private;
10993         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10994         uint32_t pf, pipesrc;
10995         int ret;
10996
10997         ret = intel_ring_begin(req, 4);
10998         if (ret)
10999                 return ret;
11000
11001         /* i965+ uses the linear or tiled offsets from the
11002          * Display Registers (which do not change across a page-flip)
11003          * so we need only reprogram the base address.
11004          */
11005         intel_ring_emit(ring, MI_DISPLAY_FLIP |
11006                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11007         intel_ring_emit(ring, fb->pitches[0]);
11008         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset |
11009                         obj->tiling_mode);
11010
11011         /* XXX Enabling the panel-fitter across page-flip is so far
11012          * untested on non-native modes, so ignore it for now.
11013          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
11014          */
11015         pf = 0;
11016         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11017         intel_ring_emit(ring, pf | pipesrc);
11018
11019         intel_mark_page_flip_active(intel_crtc->unpin_work);
11020         return 0;
11021 }
11022
11023 static int intel_gen6_queue_flip(struct drm_device *dev,
11024                                  struct drm_crtc *crtc,
11025                                  struct drm_framebuffer *fb,
11026                                  struct drm_i915_gem_object *obj,
11027                                  struct drm_i915_gem_request *req,
11028                                  uint32_t flags)
11029 {
11030         struct intel_engine_cs *ring = req->ring;
11031         struct drm_i915_private *dev_priv = dev->dev_private;
11032         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11033         uint32_t pf, pipesrc;
11034         int ret;
11035
11036         ret = intel_ring_begin(req, 4);
11037         if (ret)
11038                 return ret;
11039
11040         intel_ring_emit(ring, MI_DISPLAY_FLIP |
11041                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11042         intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
11043         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11044
11045         /* Contrary to the suggestions in the documentation,
11046          * "Enable Panel Fitter" does not seem to be required when page
11047          * flipping with a non-native mode, and worse causes a normal
11048          * modeset to fail.
11049          * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
11050          */
11051         pf = 0;
11052         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11053         intel_ring_emit(ring, pf | pipesrc);
11054
11055         intel_mark_page_flip_active(intel_crtc->unpin_work);
11056         return 0;
11057 }
11058
11059 static int intel_gen7_queue_flip(struct drm_device *dev,
11060                                  struct drm_crtc *crtc,
11061                                  struct drm_framebuffer *fb,
11062                                  struct drm_i915_gem_object *obj,
11063                                  struct drm_i915_gem_request *req,
11064                                  uint32_t flags)
11065 {
11066         struct intel_engine_cs *ring = req->ring;
11067         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11068         uint32_t plane_bit = 0;
11069         int len, ret;
11070
11071         switch (intel_crtc->plane) {
11072         case PLANE_A:
11073                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
11074                 break;
11075         case PLANE_B:
11076                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_B;
11077                 break;
11078         case PLANE_C:
11079                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_C;
11080                 break;
11081         default:
11082                 WARN_ONCE(1, "unknown plane in flip command\n");
11083                 return -ENODEV;
11084         }
11085
11086         len = 4;
11087         if (ring->id == RCS) {
11088                 len += 6;
11089                 /*
11090                  * On Gen 8, SRM is now taking an extra dword to accommodate
11091                  * 48bits addresses, and we need a NOOP for the batch size to
11092                  * stay even.
11093                  */
11094                 if (IS_GEN8(dev))
11095                         len += 2;
11096         }
11097
11098         /*
11099          * BSpec MI_DISPLAY_FLIP for IVB:
11100          * "The full packet must be contained within the same cache line."
11101          *
11102          * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same
11103          * cacheline, if we ever start emitting more commands before
11104          * the MI_DISPLAY_FLIP we may need to first emit everything else,
11105          * then do the cacheline alignment, and finally emit the
11106          * MI_DISPLAY_FLIP.
11107          */
11108         ret = intel_ring_cacheline_align(req);
11109         if (ret)
11110                 return ret;
11111
11112         ret = intel_ring_begin(req, len);
11113         if (ret)
11114                 return ret;
11115
11116         /* Unmask the flip-done completion message. Note that the bspec says that
11117          * we should do this for both the BCS and RCS, and that we must not unmask
11118          * more than one flip event at any time (or ensure that one flip message
11119          * can be sent by waiting for flip-done prior to queueing new flips).
11120          * Experimentation says that BCS works despite DERRMR masking all
11121          * flip-done completion events and that unmasking all planes at once
11122          * for the RCS also doesn't appear to drop events. Setting the DERRMR
11123          * to zero does lead to lockups within MI_DISPLAY_FLIP.
11124          */
11125         if (ring->id == RCS) {
11126                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
11127                 intel_ring_emit(ring, DERRMR);
11128                 intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
11129                                         DERRMR_PIPEB_PRI_FLIP_DONE |
11130                                         DERRMR_PIPEC_PRI_FLIP_DONE));
11131                 if (IS_GEN8(dev))
11132                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
11133                                               MI_SRM_LRM_GLOBAL_GTT);
11134                 else
11135                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM |
11136                                               MI_SRM_LRM_GLOBAL_GTT);
11137                 intel_ring_emit(ring, DERRMR);
11138                 intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
11139                 if (IS_GEN8(dev)) {
11140                         intel_ring_emit(ring, 0);
11141                         intel_ring_emit(ring, MI_NOOP);
11142                 }
11143         }
11144
11145         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
11146         intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
11147         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11148         intel_ring_emit(ring, (MI_NOOP));
11149
11150         intel_mark_page_flip_active(intel_crtc->unpin_work);
11151         return 0;
11152 }
11153
11154 static bool use_mmio_flip(struct intel_engine_cs *ring,
11155                           struct drm_i915_gem_object *obj)
11156 {
11157         /*
11158          * This is not being used for older platforms, because
11159          * non-availability of flip done interrupt forces us to use
11160          * CS flips. Older platforms derive flip done using some clever
11161          * tricks involving the flip_pending status bits and vblank irqs.
11162          * So using MMIO flips there would disrupt this mechanism.
11163          */
11164
11165         if (ring == NULL)
11166                 return true;
11167
11168         if (INTEL_INFO(ring->dev)->gen < 5)
11169                 return false;
11170
11171         if (i915.use_mmio_flip < 0)
11172                 return false;
11173         else if (i915.use_mmio_flip > 0)
11174                 return true;
11175         else if (i915.enable_execlists)
11176                 return true;
11177         else
11178                 return ring != i915_gem_request_get_ring(obj->last_write_req);
11179 }
11180
11181 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
11182                              struct intel_unpin_work *work)
11183 {
11184         struct drm_device *dev = intel_crtc->base.dev;
11185         struct drm_i915_private *dev_priv = dev->dev_private;
11186         struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
11187         const enum pipe pipe = intel_crtc->pipe;
11188         u32 ctl, stride;
11189
11190         ctl = I915_READ(PLANE_CTL(pipe, 0));
11191         ctl &= ~PLANE_CTL_TILED_MASK;
11192         switch (fb->modifier[0]) {
11193         case DRM_FORMAT_MOD_NONE:
11194                 break;
11195         case I915_FORMAT_MOD_X_TILED:
11196                 ctl |= PLANE_CTL_TILED_X;
11197                 break;
11198         case I915_FORMAT_MOD_Y_TILED:
11199                 ctl |= PLANE_CTL_TILED_Y;
11200                 break;
11201         case I915_FORMAT_MOD_Yf_TILED:
11202                 ctl |= PLANE_CTL_TILED_YF;
11203                 break;
11204         default:
11205                 MISSING_CASE(fb->modifier[0]);
11206         }
11207
11208         /*
11209          * The stride is either expressed as a multiple of 64 bytes chunks for
11210          * linear buffers or in number of tiles for tiled buffers.
11211          */
11212         stride = fb->pitches[0] /
11213                  intel_fb_stride_alignment(dev, fb->modifier[0],
11214                                            fb->pixel_format);
11215
11216         /*
11217          * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
11218          * PLANE_SURF updates, the update is then guaranteed to be atomic.
11219          */
11220         I915_WRITE(PLANE_CTL(pipe, 0), ctl);
11221         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
11222
11223         I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset);
11224         POSTING_READ(PLANE_SURF(pipe, 0));
11225 }
11226
11227 static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
11228                              struct intel_unpin_work *work)
11229 {
11230         struct drm_device *dev = intel_crtc->base.dev;
11231         struct drm_i915_private *dev_priv = dev->dev_private;
11232         struct intel_framebuffer *intel_fb =
11233                 to_intel_framebuffer(intel_crtc->base.primary->fb);
11234         struct drm_i915_gem_object *obj = intel_fb->obj;
11235         u32 dspcntr;
11236         u32 reg;
11237
11238         reg = DSPCNTR(intel_crtc->plane);
11239         dspcntr = I915_READ(reg);
11240
11241         if (obj->tiling_mode != I915_TILING_NONE)
11242                 dspcntr |= DISPPLANE_TILED;
11243         else
11244                 dspcntr &= ~DISPPLANE_TILED;
11245
11246         I915_WRITE(reg, dspcntr);
11247
11248         I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset);
11249         POSTING_READ(DSPSURF(intel_crtc->plane));
11250 }
11251
11252 /*
11253  * XXX: This is the temporary way to update the plane registers until we get
11254  * around to using the usual plane update functions for MMIO flips
11255  */
11256 static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip)
11257 {
11258         struct intel_crtc *crtc = mmio_flip->crtc;
11259         struct intel_unpin_work *work;
11260
11261         spin_lock_irq(&crtc->base.dev->event_lock);
11262         work = crtc->unpin_work;
11263         spin_unlock_irq(&crtc->base.dev->event_lock);
11264         if (work == NULL)
11265                 return;
11266
11267         intel_mark_page_flip_active(work);
11268
11269         intel_pipe_update_start(crtc);
11270
11271         if (INTEL_INFO(mmio_flip->i915)->gen >= 9)
11272                 skl_do_mmio_flip(crtc, work);
11273         else
11274                 /* use_mmio_flip() retricts MMIO flips to ilk+ */
11275                 ilk_do_mmio_flip(crtc, work);
11276
11277         intel_pipe_update_end(crtc);
11278 }
11279
11280 static void intel_mmio_flip_work_func(struct work_struct *work)
11281 {
11282         struct intel_mmio_flip *mmio_flip =
11283                 container_of(work, struct intel_mmio_flip, work);
11284
11285         if (mmio_flip->req) {
11286                 WARN_ON(__i915_wait_request(mmio_flip->req,
11287                                             mmio_flip->crtc->reset_counter,
11288                                             false, NULL,
11289                                             &mmio_flip->i915->rps.mmioflips));
11290                 i915_gem_request_unreference__unlocked(mmio_flip->req);
11291         }
11292
11293         intel_do_mmio_flip(mmio_flip);
11294         kfree(mmio_flip);
11295 }
11296
11297 static int intel_queue_mmio_flip(struct drm_device *dev,
11298                                  struct drm_crtc *crtc,
11299                                  struct drm_framebuffer *fb,
11300                                  struct drm_i915_gem_object *obj,
11301                                  struct intel_engine_cs *ring,
11302                                  uint32_t flags)
11303 {
11304         struct intel_mmio_flip *mmio_flip;
11305
11306         mmio_flip = kmalloc(sizeof(*mmio_flip), GFP_KERNEL);
11307         if (mmio_flip == NULL)
11308                 return -ENOMEM;
11309
11310         mmio_flip->i915 = to_i915(dev);
11311         mmio_flip->req = i915_gem_request_reference(obj->last_write_req);
11312         mmio_flip->crtc = to_intel_crtc(crtc);
11313
11314         INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func);
11315         schedule_work(&mmio_flip->work);
11316
11317         return 0;
11318 }
11319
11320 static int intel_default_queue_flip(struct drm_device *dev,
11321                                     struct drm_crtc *crtc,
11322                                     struct drm_framebuffer *fb,
11323                                     struct drm_i915_gem_object *obj,
11324                                     struct drm_i915_gem_request *req,
11325                                     uint32_t flags)
11326 {
11327         return -ENODEV;
11328 }
11329
11330 static bool __intel_pageflip_stall_check(struct drm_device *dev,
11331                                          struct drm_crtc *crtc)
11332 {
11333         struct drm_i915_private *dev_priv = dev->dev_private;
11334         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11335         struct intel_unpin_work *work = intel_crtc->unpin_work;
11336         u32 addr;
11337
11338         if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
11339                 return true;
11340
11341         if (atomic_read(&work->pending) < INTEL_FLIP_PENDING)
11342                 return false;
11343
11344         if (!work->enable_stall_check)
11345                 return false;
11346
11347         if (work->flip_ready_vblank == 0) {
11348                 if (work->flip_queued_req &&
11349                     !i915_gem_request_completed(work->flip_queued_req, true))
11350                         return false;
11351
11352                 work->flip_ready_vblank = drm_crtc_vblank_count(crtc);
11353         }
11354
11355         if (drm_crtc_vblank_count(crtc) - work->flip_ready_vblank < 3)
11356                 return false;
11357
11358         /* Potential stall - if we see that the flip has happened,
11359          * assume a missed interrupt. */
11360         if (INTEL_INFO(dev)->gen >= 4)
11361                 addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
11362         else
11363                 addr = I915_READ(DSPADDR(intel_crtc->plane));
11364
11365         /* There is a potential issue here with a false positive after a flip
11366          * to the same address. We could address this by checking for a
11367          * non-incrementing frame counter.
11368          */
11369         return addr == work->gtt_offset;
11370 }
11371
11372 void intel_check_page_flip(struct drm_device *dev, int pipe)
11373 {
11374         struct drm_i915_private *dev_priv = dev->dev_private;
11375         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11376         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11377         struct intel_unpin_work *work;
11378
11379         WARN_ON_NONRT(!in_interrupt());
11380
11381         if (crtc == NULL)
11382                 return;
11383
11384         spin_lock(&dev->event_lock);
11385         work = intel_crtc->unpin_work;
11386         if (work != NULL && __intel_pageflip_stall_check(dev, crtc)) {
11387                 WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
11388                          work->flip_queued_vblank, drm_vblank_count(dev, pipe));
11389                 page_flip_completed(intel_crtc);
11390                 work = NULL;
11391         }
11392         if (work != NULL &&
11393             drm_vblank_count(dev, pipe) - work->flip_queued_vblank > 1)
11394                 intel_queue_rps_boost_for_request(dev, work->flip_queued_req);
11395         spin_unlock(&dev->event_lock);
11396 }
11397
11398 static int intel_crtc_page_flip(struct drm_crtc *crtc,
11399                                 struct drm_framebuffer *fb,
11400                                 struct drm_pending_vblank_event *event,
11401                                 uint32_t page_flip_flags)
11402 {
11403         struct drm_device *dev = crtc->dev;
11404         struct drm_i915_private *dev_priv = dev->dev_private;
11405         struct drm_framebuffer *old_fb = crtc->primary->fb;
11406         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11407         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11408         struct drm_plane *primary = crtc->primary;
11409         enum pipe pipe = intel_crtc->pipe;
11410         struct intel_unpin_work *work;
11411         struct intel_engine_cs *ring;
11412         bool mmio_flip;
11413         struct drm_i915_gem_request *request = NULL;
11414         int ret;
11415
11416         /*
11417          * drm_mode_page_flip_ioctl() should already catch this, but double
11418          * check to be safe.  In the future we may enable pageflipping from
11419          * a disabled primary plane.
11420          */
11421         if (WARN_ON(intel_fb_obj(old_fb) == NULL))
11422                 return -EBUSY;
11423
11424         /* Can't change pixel format via MI display flips. */
11425         if (fb->pixel_format != crtc->primary->fb->pixel_format)
11426                 return -EINVAL;
11427
11428         /*
11429          * TILEOFF/LINOFF registers can't be changed via MI display flips.
11430          * Note that pitch changes could also affect these register.
11431          */
11432         if (INTEL_INFO(dev)->gen > 3 &&
11433             (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
11434              fb->pitches[0] != crtc->primary->fb->pitches[0]))
11435                 return -EINVAL;
11436
11437         if (i915_terminally_wedged(&dev_priv->gpu_error))
11438                 goto out_hang;
11439
11440         work = kzalloc(sizeof(*work), GFP_KERNEL);
11441         if (work == NULL)
11442                 return -ENOMEM;
11443
11444         work->event = event;
11445         work->crtc = crtc;
11446         work->old_fb = old_fb;
11447         INIT_WORK(&work->work, intel_unpin_work_fn);
11448
11449         ret = drm_crtc_vblank_get(crtc);
11450         if (ret)
11451                 goto free_work;
11452
11453         /* We borrow the event spin lock for protecting unpin_work */
11454         spin_lock_irq(&dev->event_lock);
11455         if (intel_crtc->unpin_work) {
11456                 /* Before declaring the flip queue wedged, check if
11457                  * the hardware completed the operation behind our backs.
11458                  */
11459                 if (__intel_pageflip_stall_check(dev, crtc)) {
11460                         DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
11461                         page_flip_completed(intel_crtc);
11462                 } else {
11463                         DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
11464                         spin_unlock_irq(&dev->event_lock);
11465
11466                         drm_crtc_vblank_put(crtc);
11467                         kfree(work);
11468                         return -EBUSY;
11469                 }
11470         }
11471         intel_crtc->unpin_work = work;
11472         spin_unlock_irq(&dev->event_lock);
11473
11474         if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
11475                 flush_workqueue(dev_priv->wq);
11476
11477         /* Reference the objects for the scheduled work. */
11478         drm_framebuffer_reference(work->old_fb);
11479         drm_gem_object_reference(&obj->base);
11480
11481         crtc->primary->fb = fb;
11482         update_state_fb(crtc->primary);
11483
11484         work->pending_flip_obj = obj;
11485
11486         ret = i915_mutex_lock_interruptible(dev);
11487         if (ret)
11488                 goto cleanup;
11489
11490         atomic_inc(&intel_crtc->unpin_work_count);
11491         intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
11492
11493         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
11494                 work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
11495
11496         if (IS_VALLEYVIEW(dev)) {
11497                 ring = &dev_priv->ring[BCS];
11498                 if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
11499                         /* vlv: DISPLAY_FLIP fails to change tiling */
11500                         ring = NULL;
11501         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
11502                 ring = &dev_priv->ring[BCS];
11503         } else if (INTEL_INFO(dev)->gen >= 7) {
11504                 ring = i915_gem_request_get_ring(obj->last_write_req);
11505                 if (ring == NULL || ring->id != RCS)
11506                         ring = &dev_priv->ring[BCS];
11507         } else {
11508                 ring = &dev_priv->ring[RCS];
11509         }
11510
11511         mmio_flip = use_mmio_flip(ring, obj);
11512
11513         /* When using CS flips, we want to emit semaphores between rings.
11514          * However, when using mmio flips we will create a task to do the
11515          * synchronisation, so all we want here is to pin the framebuffer
11516          * into the display plane and skip any waits.
11517          */
11518         ret = intel_pin_and_fence_fb_obj(crtc->primary, fb,
11519                                          crtc->primary->state,
11520                                          mmio_flip ? i915_gem_request_get_ring(obj->last_write_req) : ring, &request);
11521         if (ret)
11522                 goto cleanup_pending;
11523
11524         work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
11525                                                   obj, 0);
11526         work->gtt_offset += intel_crtc->dspaddr_offset;
11527
11528         if (mmio_flip) {
11529                 ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
11530                                             page_flip_flags);
11531                 if (ret)
11532                         goto cleanup_unpin;
11533
11534                 i915_gem_request_assign(&work->flip_queued_req,
11535                                         obj->last_write_req);
11536         } else {
11537                 if (!request) {
11538                         ret = i915_gem_request_alloc(ring, ring->default_context, &request);
11539                         if (ret)
11540                                 goto cleanup_unpin;
11541                 }
11542
11543                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
11544                                                    page_flip_flags);
11545                 if (ret)
11546                         goto cleanup_unpin;
11547
11548                 i915_gem_request_assign(&work->flip_queued_req, request);
11549         }
11550
11551         if (request)
11552                 i915_add_request_no_flush(request);
11553
11554         work->flip_queued_vblank = drm_crtc_vblank_count(crtc);
11555         work->enable_stall_check = true;
11556
11557         i915_gem_track_fb(intel_fb_obj(work->old_fb), obj,
11558                           to_intel_plane(primary)->frontbuffer_bit);
11559         mutex_unlock(&dev->struct_mutex);
11560
11561         intel_fbc_disable_crtc(intel_crtc);
11562         intel_frontbuffer_flip_prepare(dev,
11563                                        to_intel_plane(primary)->frontbuffer_bit);
11564
11565         trace_i915_flip_request(intel_crtc->plane, obj);
11566
11567         return 0;
11568
11569 cleanup_unpin:
11570         intel_unpin_fb_obj(fb, crtc->primary->state);
11571 cleanup_pending:
11572         if (request)
11573                 i915_gem_request_cancel(request);
11574         atomic_dec(&intel_crtc->unpin_work_count);
11575         mutex_unlock(&dev->struct_mutex);
11576 cleanup:
11577         crtc->primary->fb = old_fb;
11578         update_state_fb(crtc->primary);
11579
11580         drm_gem_object_unreference_unlocked(&obj->base);
11581         drm_framebuffer_unreference(work->old_fb);
11582
11583         spin_lock_irq(&dev->event_lock);
11584         intel_crtc->unpin_work = NULL;
11585         spin_unlock_irq(&dev->event_lock);
11586
11587         drm_crtc_vblank_put(crtc);
11588 free_work:
11589         kfree(work);
11590
11591         if (ret == -EIO) {
11592                 struct drm_atomic_state *state;
11593                 struct drm_plane_state *plane_state;
11594
11595 out_hang:
11596                 state = drm_atomic_state_alloc(dev);
11597                 if (!state)
11598                         return -ENOMEM;
11599                 state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
11600
11601 retry:
11602                 plane_state = drm_atomic_get_plane_state(state, primary);
11603                 ret = PTR_ERR_OR_ZERO(plane_state);
11604                 if (!ret) {
11605                         drm_atomic_set_fb_for_plane(plane_state, fb);
11606
11607                         ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
11608                         if (!ret)
11609                                 ret = drm_atomic_commit(state);
11610                 }
11611
11612                 if (ret == -EDEADLK) {
11613                         drm_modeset_backoff(state->acquire_ctx);
11614                         drm_atomic_state_clear(state);
11615                         goto retry;
11616                 }
11617
11618                 if (ret)
11619                         drm_atomic_state_free(state);
11620
11621                 if (ret == 0 && event) {
11622                         spin_lock_irq(&dev->event_lock);
11623                         drm_send_vblank_event(dev, pipe, event);
11624                         spin_unlock_irq(&dev->event_lock);
11625                 }
11626         }
11627         return ret;
11628 }
11629
11630
11631 /**
11632  * intel_wm_need_update - Check whether watermarks need updating
11633  * @plane: drm plane
11634  * @state: new plane state
11635  *
11636  * Check current plane state versus the new one to determine whether
11637  * watermarks need to be recalculated.
11638  *
11639  * Returns true or false.
11640  */
11641 static bool intel_wm_need_update(struct drm_plane *plane,
11642                                  struct drm_plane_state *state)
11643 {
11644         /* Update watermarks on tiling changes. */
11645         if (!plane->state->fb || !state->fb ||
11646             plane->state->fb->modifier[0] != state->fb->modifier[0] ||
11647             plane->state->rotation != state->rotation)
11648                 return true;
11649
11650         if (plane->state->crtc_w != state->crtc_w)
11651                 return true;
11652
11653         return false;
11654 }
11655
11656 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
11657                                     struct drm_plane_state *plane_state)
11658 {
11659         struct drm_crtc *crtc = crtc_state->crtc;
11660         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11661         struct drm_plane *plane = plane_state->plane;
11662         struct drm_device *dev = crtc->dev;
11663         struct drm_i915_private *dev_priv = dev->dev_private;
11664         struct intel_plane_state *old_plane_state =
11665                 to_intel_plane_state(plane->state);
11666         int idx = intel_crtc->base.base.id, ret;
11667         int i = drm_plane_index(plane);
11668         bool mode_changed = needs_modeset(crtc_state);
11669         bool was_crtc_enabled = crtc->state->active;
11670         bool is_crtc_enabled = crtc_state->active;
11671
11672         bool turn_off, turn_on, visible, was_visible;
11673         struct drm_framebuffer *fb = plane_state->fb;
11674
11675         if (crtc_state && INTEL_INFO(dev)->gen >= 9 &&
11676             plane->type != DRM_PLANE_TYPE_CURSOR) {
11677                 ret = skl_update_scaler_plane(
11678                         to_intel_crtc_state(crtc_state),
11679                         to_intel_plane_state(plane_state));
11680                 if (ret)
11681                         return ret;
11682         }
11683
11684         /*
11685          * Disabling a plane is always okay; we just need to update
11686          * fb tracking in a special way since cleanup_fb() won't
11687          * get called by the plane helpers.
11688          */
11689         if (old_plane_state->base.fb && !fb)
11690                 intel_crtc->atomic.disabled_planes |= 1 << i;
11691
11692         was_visible = old_plane_state->visible;
11693         visible = to_intel_plane_state(plane_state)->visible;
11694
11695         if (!was_crtc_enabled && WARN_ON(was_visible))
11696                 was_visible = false;
11697
11698         if (!is_crtc_enabled && WARN_ON(visible))
11699                 visible = false;
11700
11701         if (!was_visible && !visible)
11702                 return 0;
11703
11704         turn_off = was_visible && (!visible || mode_changed);
11705         turn_on = visible && (!was_visible || mode_changed);
11706
11707         DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx,
11708                          plane->base.id, fb ? fb->base.id : -1);
11709
11710         DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n",
11711                          plane->base.id, was_visible, visible,
11712                          turn_off, turn_on, mode_changed);
11713
11714         if (turn_on) {
11715                 intel_crtc->atomic.update_wm_pre = true;
11716                 /* must disable cxsr around plane enable/disable */
11717                 if (plane->type != DRM_PLANE_TYPE_CURSOR) {
11718                         intel_crtc->atomic.disable_cxsr = true;
11719                         /* to potentially re-enable cxsr */
11720                         intel_crtc->atomic.wait_vblank = true;
11721                         intel_crtc->atomic.update_wm_post = true;
11722                 }
11723         } else if (turn_off) {
11724                 intel_crtc->atomic.update_wm_post = true;
11725                 /* must disable cxsr around plane enable/disable */
11726                 if (plane->type != DRM_PLANE_TYPE_CURSOR) {
11727                         if (is_crtc_enabled)
11728                                 intel_crtc->atomic.wait_vblank = true;
11729                         intel_crtc->atomic.disable_cxsr = true;
11730                 }
11731         } else if (intel_wm_need_update(plane, plane_state)) {
11732                 intel_crtc->atomic.update_wm_pre = true;
11733         }
11734
11735         if (visible || was_visible)
11736                 intel_crtc->atomic.fb_bits |=
11737                         to_intel_plane(plane)->frontbuffer_bit;
11738
11739         switch (plane->type) {
11740         case DRM_PLANE_TYPE_PRIMARY:
11741                 intel_crtc->atomic.wait_for_flips = true;
11742                 intel_crtc->atomic.pre_disable_primary = turn_off;
11743                 intel_crtc->atomic.post_enable_primary = turn_on;
11744
11745                 if (turn_off) {
11746                         /*
11747                          * FIXME: Actually if we will still have any other
11748                          * plane enabled on the pipe we could let IPS enabled
11749                          * still, but for now lets consider that when we make
11750                          * primary invisible by setting DSPCNTR to 0 on
11751                          * update_primary_plane function IPS needs to be
11752                          * disable.
11753                          */
11754                         intel_crtc->atomic.disable_ips = true;
11755
11756                         intel_crtc->atomic.disable_fbc = true;
11757                 }
11758
11759                 /*
11760                  * FBC does not work on some platforms for rotated
11761                  * planes, so disable it when rotation is not 0 and
11762                  * update it when rotation is set back to 0.
11763                  *
11764                  * FIXME: This is redundant with the fbc update done in
11765                  * the primary plane enable function except that that
11766                  * one is done too late. We eventually need to unify
11767                  * this.
11768                  */
11769
11770                 if (visible &&
11771                     INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
11772                     dev_priv->fbc.crtc == intel_crtc &&
11773                     plane_state->rotation != BIT(DRM_ROTATE_0))
11774                         intel_crtc->atomic.disable_fbc = true;
11775
11776                 /*
11777                  * BDW signals flip done immediately if the plane
11778                  * is disabled, even if the plane enable is already
11779                  * armed to occur at the next vblank :(
11780                  */
11781                 if (turn_on && IS_BROADWELL(dev))
11782                         intel_crtc->atomic.wait_vblank = true;
11783
11784                 intel_crtc->atomic.update_fbc |= visible || mode_changed;
11785                 break;
11786         case DRM_PLANE_TYPE_CURSOR:
11787                 break;
11788         case DRM_PLANE_TYPE_OVERLAY:
11789                 if (turn_off && !mode_changed) {
11790                         intel_crtc->atomic.wait_vblank = true;
11791                         intel_crtc->atomic.update_sprite_watermarks |=
11792                                 1 << i;
11793                 }
11794         }
11795         return 0;
11796 }
11797
11798 static bool encoders_cloneable(const struct intel_encoder *a,
11799                                const struct intel_encoder *b)
11800 {
11801         /* masks could be asymmetric, so check both ways */
11802         return a == b || (a->cloneable & (1 << b->type) &&
11803                           b->cloneable & (1 << a->type));
11804 }
11805
11806 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11807                                          struct intel_crtc *crtc,
11808                                          struct intel_encoder *encoder)
11809 {
11810         struct intel_encoder *source_encoder;
11811         struct drm_connector *connector;
11812         struct drm_connector_state *connector_state;
11813         int i;
11814
11815         for_each_connector_in_state(state, connector, connector_state, i) {
11816                 if (connector_state->crtc != &crtc->base)
11817                         continue;
11818
11819                 source_encoder =
11820                         to_intel_encoder(connector_state->best_encoder);
11821                 if (!encoders_cloneable(encoder, source_encoder))
11822                         return false;
11823         }
11824
11825         return true;
11826 }
11827
11828 static bool check_encoder_cloning(struct drm_atomic_state *state,
11829                                   struct intel_crtc *crtc)
11830 {
11831         struct intel_encoder *encoder;
11832         struct drm_connector *connector;
11833         struct drm_connector_state *connector_state;
11834         int i;
11835
11836         for_each_connector_in_state(state, connector, connector_state, i) {
11837                 if (connector_state->crtc != &crtc->base)
11838                         continue;
11839
11840                 encoder = to_intel_encoder(connector_state->best_encoder);
11841                 if (!check_single_encoder_cloning(state, crtc, encoder))
11842                         return false;
11843         }
11844
11845         return true;
11846 }
11847
11848 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11849                                    struct drm_crtc_state *crtc_state)
11850 {
11851         struct drm_device *dev = crtc->dev;
11852         struct drm_i915_private *dev_priv = dev->dev_private;
11853         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11854         struct intel_crtc_state *pipe_config =
11855                 to_intel_crtc_state(crtc_state);
11856         struct drm_atomic_state *state = crtc_state->state;
11857         int ret;
11858         bool mode_changed = needs_modeset(crtc_state);
11859
11860         if (mode_changed && !check_encoder_cloning(state, intel_crtc)) {
11861                 DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11862                 return -EINVAL;
11863         }
11864
11865         if (mode_changed && !crtc_state->active)
11866                 intel_crtc->atomic.update_wm_post = true;
11867
11868         if (mode_changed && crtc_state->enable &&
11869             dev_priv->display.crtc_compute_clock &&
11870             !WARN_ON(pipe_config->shared_dpll != DPLL_ID_PRIVATE)) {
11871                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11872                                                            pipe_config);
11873                 if (ret)
11874                         return ret;
11875         }
11876
11877         ret = 0;
11878         if (INTEL_INFO(dev)->gen >= 9) {
11879                 if (mode_changed)
11880                         ret = skl_update_scaler_crtc(pipe_config);
11881
11882                 if (!ret)
11883                         ret = intel_atomic_setup_scalers(dev, intel_crtc,
11884                                                          pipe_config);
11885         }
11886
11887         return ret;
11888 }
11889
11890 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11891         .mode_set_base_atomic = intel_pipe_set_base_atomic,
11892         .load_lut = intel_crtc_load_lut,
11893         .atomic_begin = intel_begin_crtc_commit,
11894         .atomic_flush = intel_finish_crtc_commit,
11895         .atomic_check = intel_crtc_atomic_check,
11896 };
11897
11898 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11899 {
11900         struct intel_connector *connector;
11901
11902         for_each_intel_connector(dev, connector) {
11903                 if (connector->base.encoder) {
11904                         connector->base.state->best_encoder =
11905                                 connector->base.encoder;
11906                         connector->base.state->crtc =
11907                                 connector->base.encoder->crtc;
11908                 } else {
11909                         connector->base.state->best_encoder = NULL;
11910                         connector->base.state->crtc = NULL;
11911                 }
11912         }
11913 }
11914
11915 static void
11916 connected_sink_compute_bpp(struct intel_connector *connector,
11917                            struct intel_crtc_state *pipe_config)
11918 {
11919         int bpp = pipe_config->pipe_bpp;
11920
11921         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
11922                 connector->base.base.id,
11923                 connector->base.name);
11924
11925         /* Don't use an invalid EDID bpc value */
11926         if (connector->base.display_info.bpc &&
11927             connector->base.display_info.bpc * 3 < bpp) {
11928                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
11929                               bpp, connector->base.display_info.bpc*3);
11930                 pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
11931         }
11932
11933         /* Clamp bpp to default limit on screens without EDID 1.4 */
11934         if (connector->base.display_info.bpc == 0) {
11935                 int type = connector->base.connector_type;
11936                 int clamp_bpp = 24;
11937
11938                 /* Fall back to 18 bpp when DP sink capability is unknown. */
11939                 if (type == DRM_MODE_CONNECTOR_DisplayPort ||
11940                     type == DRM_MODE_CONNECTOR_eDP)
11941                         clamp_bpp = 18;
11942
11943                 if (bpp > clamp_bpp) {
11944                         DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of %d\n",
11945                                       bpp, clamp_bpp);
11946                         pipe_config->pipe_bpp = clamp_bpp;
11947                 }
11948         }
11949 }
11950
11951 static int
11952 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11953                           struct intel_crtc_state *pipe_config)
11954 {
11955         struct drm_device *dev = crtc->base.dev;
11956         struct drm_atomic_state *state;
11957         struct drm_connector *connector;
11958         struct drm_connector_state *connector_state;
11959         int bpp, i;
11960
11961         if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)))
11962                 bpp = 10*3;
11963         else if (INTEL_INFO(dev)->gen >= 5)
11964                 bpp = 12*3;
11965         else
11966                 bpp = 8*3;
11967
11968
11969         pipe_config->pipe_bpp = bpp;
11970
11971         state = pipe_config->base.state;
11972
11973         /* Clamp display bpp to EDID value */
11974         for_each_connector_in_state(state, connector, connector_state, i) {
11975                 if (connector_state->crtc != &crtc->base)
11976                         continue;
11977
11978                 connected_sink_compute_bpp(to_intel_connector(connector),
11979                                            pipe_config);
11980         }
11981
11982         return bpp;
11983 }
11984
11985 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11986 {
11987         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11988                         "type: 0x%x flags: 0x%x\n",
11989                 mode->crtc_clock,
11990                 mode->crtc_hdisplay, mode->crtc_hsync_start,
11991                 mode->crtc_hsync_end, mode->crtc_htotal,
11992                 mode->crtc_vdisplay, mode->crtc_vsync_start,
11993                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
11994 }
11995
11996 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11997                                    struct intel_crtc_state *pipe_config,
11998                                    const char *context)
11999 {
12000         struct drm_device *dev = crtc->base.dev;
12001         struct drm_plane *plane;
12002         struct intel_plane *intel_plane;
12003         struct intel_plane_state *state;
12004         struct drm_framebuffer *fb;
12005
12006         DRM_DEBUG_KMS("[CRTC:%d]%s config %p for pipe %c\n", crtc->base.base.id,
12007                       context, pipe_config, pipe_name(crtc->pipe));
12008
12009         DRM_DEBUG_KMS("cpu_transcoder: %c\n", transcoder_name(pipe_config->cpu_transcoder));
12010         DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
12011                       pipe_config->pipe_bpp, pipe_config->dither);
12012         DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12013                       pipe_config->has_pch_encoder,
12014                       pipe_config->fdi_lanes,
12015                       pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
12016                       pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
12017                       pipe_config->fdi_m_n.tu);
12018         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12019                       pipe_config->has_dp_encoder,
12020                       pipe_config->lane_count,
12021                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
12022                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
12023                       pipe_config->dp_m_n.tu);
12024
12025         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
12026                       pipe_config->has_dp_encoder,
12027                       pipe_config->lane_count,
12028                       pipe_config->dp_m2_n2.gmch_m,
12029                       pipe_config->dp_m2_n2.gmch_n,
12030                       pipe_config->dp_m2_n2.link_m,
12031                       pipe_config->dp_m2_n2.link_n,
12032                       pipe_config->dp_m2_n2.tu);
12033
12034         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
12035                       pipe_config->has_audio,
12036                       pipe_config->has_infoframe);
12037
12038         DRM_DEBUG_KMS("requested mode:\n");
12039         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12040         DRM_DEBUG_KMS("adjusted mode:\n");
12041         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12042         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12043         DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock);
12044         DRM_DEBUG_KMS("pipe src size: %dx%d\n",
12045                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
12046         DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12047                       crtc->num_scalers,
12048                       pipe_config->scaler_state.scaler_users,
12049                       pipe_config->scaler_state.scaler_id);
12050         DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12051                       pipe_config->gmch_pfit.control,
12052                       pipe_config->gmch_pfit.pgm_ratios,
12053                       pipe_config->gmch_pfit.lvds_border_bits);
12054         DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
12055                       pipe_config->pch_pfit.pos,
12056                       pipe_config->pch_pfit.size,
12057                       pipe_config->pch_pfit.enabled ? "enabled" : "disabled");
12058         DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
12059         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
12060
12061         if (IS_BROXTON(dev)) {
12062                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
12063                               "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
12064                               "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
12065                               pipe_config->ddi_pll_sel,
12066                               pipe_config->dpll_hw_state.ebb0,
12067                               pipe_config->dpll_hw_state.ebb4,
12068                               pipe_config->dpll_hw_state.pll0,
12069                               pipe_config->dpll_hw_state.pll1,
12070                               pipe_config->dpll_hw_state.pll2,
12071                               pipe_config->dpll_hw_state.pll3,
12072                               pipe_config->dpll_hw_state.pll6,
12073                               pipe_config->dpll_hw_state.pll8,
12074                               pipe_config->dpll_hw_state.pll9,
12075                               pipe_config->dpll_hw_state.pll10,
12076                               pipe_config->dpll_hw_state.pcsdw12);
12077         } else if (IS_SKYLAKE(dev)) {
12078                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
12079                               "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
12080                               pipe_config->ddi_pll_sel,
12081                               pipe_config->dpll_hw_state.ctrl1,
12082                               pipe_config->dpll_hw_state.cfgcr1,
12083                               pipe_config->dpll_hw_state.cfgcr2);
12084         } else if (HAS_DDI(dev)) {
12085                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
12086                               pipe_config->ddi_pll_sel,
12087                               pipe_config->dpll_hw_state.wrpll,
12088                               pipe_config->dpll_hw_state.spll);
12089         } else {
12090                 DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
12091                               "fp0: 0x%x, fp1: 0x%x\n",
12092                               pipe_config->dpll_hw_state.dpll,
12093                               pipe_config->dpll_hw_state.dpll_md,
12094                               pipe_config->dpll_hw_state.fp0,
12095                               pipe_config->dpll_hw_state.fp1);
12096         }
12097
12098         DRM_DEBUG_KMS("planes on this crtc\n");
12099         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
12100                 intel_plane = to_intel_plane(plane);
12101                 if (intel_plane->pipe != crtc->pipe)
12102                         continue;
12103
12104                 state = to_intel_plane_state(plane->state);
12105                 fb = state->base.fb;
12106                 if (!fb) {
12107                         DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d "
12108                                 "disabled, scaler_id = %d\n",
12109                                 plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12110                                 plane->base.id, intel_plane->pipe,
12111                                 (crtc->base.primary == plane) ? 0 : intel_plane->plane + 1,
12112                                 drm_plane_index(plane), state->scaler_id);
12113                         continue;
12114                 }
12115
12116                 DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d enabled",
12117                         plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12118                         plane->base.id, intel_plane->pipe,
12119                         crtc->base.primary == plane ? 0 : intel_plane->plane + 1,
12120                         drm_plane_index(plane));
12121                 DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = 0x%x",
12122                         fb->base.id, fb->width, fb->height, fb->pixel_format);
12123                 DRM_DEBUG_KMS("\tscaler:%d src (%u, %u) %ux%u dst (%u, %u) %ux%u\n",
12124                         state->scaler_id,
12125                         state->src.x1 >> 16, state->src.y1 >> 16,
12126                         drm_rect_width(&state->src) >> 16,
12127                         drm_rect_height(&state->src) >> 16,
12128                         state->dst.x1, state->dst.y1,
12129                         drm_rect_width(&state->dst), drm_rect_height(&state->dst));
12130         }
12131 }
12132
12133 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
12134 {
12135         struct drm_device *dev = state->dev;
12136         struct drm_connector *connector;
12137         unsigned int used_ports = 0;
12138
12139         /*
12140          * Walk the connector list instead of the encoder
12141          * list to detect the problem on ddi platforms
12142          * where there's just one encoder per digital port.
12143          */
12144         drm_for_each_connector(connector, dev) {
12145                 struct drm_connector_state *connector_state;
12146                 struct intel_encoder *encoder;
12147
12148                 connector_state = drm_atomic_get_existing_connector_state(state, connector);
12149                 if (!connector_state)
12150                         connector_state = connector->state;
12151
12152                 if (!connector_state->best_encoder)
12153                         continue;
12154
12155                 encoder = to_intel_encoder(connector_state->best_encoder);
12156
12157                 WARN_ON(!connector_state->crtc);
12158
12159                 switch (encoder->type) {
12160                         unsigned int port_mask;
12161                 case INTEL_OUTPUT_UNKNOWN:
12162                         if (WARN_ON(!HAS_DDI(dev)))
12163                                 break;
12164                 case INTEL_OUTPUT_DISPLAYPORT:
12165                 case INTEL_OUTPUT_HDMI:
12166                 case INTEL_OUTPUT_EDP:
12167                         port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
12168
12169                         /* the same port mustn't appear more than once */
12170                         if (used_ports & port_mask)
12171                                 return false;
12172
12173                         used_ports |= port_mask;
12174                 default:
12175                         break;
12176                 }
12177         }
12178
12179         return true;
12180 }
12181
12182 static void
12183 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12184 {
12185         struct drm_crtc_state tmp_state;
12186         struct intel_crtc_scaler_state scaler_state;
12187         struct intel_dpll_hw_state dpll_hw_state;
12188         enum intel_dpll_id shared_dpll;
12189         uint32_t ddi_pll_sel;
12190         bool force_thru;
12191
12192         /* FIXME: before the switch to atomic started, a new pipe_config was
12193          * kzalloc'd. Code that depends on any field being zero should be
12194          * fixed, so that the crtc_state can be safely duplicated. For now,
12195          * only fields that are know to not cause problems are preserved. */
12196
12197         tmp_state = crtc_state->base;
12198         scaler_state = crtc_state->scaler_state;
12199         shared_dpll = crtc_state->shared_dpll;
12200         dpll_hw_state = crtc_state->dpll_hw_state;
12201         ddi_pll_sel = crtc_state->ddi_pll_sel;
12202         force_thru = crtc_state->pch_pfit.force_thru;
12203
12204         memset(crtc_state, 0, sizeof *crtc_state);
12205
12206         crtc_state->base = tmp_state;
12207         crtc_state->scaler_state = scaler_state;
12208         crtc_state->shared_dpll = shared_dpll;
12209         crtc_state->dpll_hw_state = dpll_hw_state;
12210         crtc_state->ddi_pll_sel = ddi_pll_sel;
12211         crtc_state->pch_pfit.force_thru = force_thru;
12212 }
12213
12214 static int
12215 intel_modeset_pipe_config(struct drm_crtc *crtc,
12216                           struct intel_crtc_state *pipe_config)
12217 {
12218         struct drm_atomic_state *state = pipe_config->base.state;
12219         struct intel_encoder *encoder;
12220         struct drm_connector *connector;
12221         struct drm_connector_state *connector_state;
12222         int base_bpp, ret = -EINVAL;
12223         int i;
12224         bool retry = true;
12225
12226         clear_intel_crtc_state(pipe_config);
12227
12228         pipe_config->cpu_transcoder =
12229                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12230
12231         /*
12232          * Sanitize sync polarity flags based on requested ones. If neither
12233          * positive or negative polarity is requested, treat this as meaning
12234          * negative polarity.
12235          */
12236         if (!(pipe_config->base.adjusted_mode.flags &
12237               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12238                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12239
12240         if (!(pipe_config->base.adjusted_mode.flags &
12241               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12242                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12243
12244         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12245                                              pipe_config);
12246         if (base_bpp < 0)
12247                 goto fail;
12248
12249         /*
12250          * Determine the real pipe dimensions. Note that stereo modes can
12251          * increase the actual pipe size due to the frame doubling and
12252          * insertion of additional space for blanks between the frame. This
12253          * is stored in the crtc timings. We use the requested mode to do this
12254          * computation to clearly distinguish it from the adjusted mode, which
12255          * can be changed by the connectors in the below retry loop.
12256          */
12257         drm_crtc_get_hv_timing(&pipe_config->base.mode,
12258                                &pipe_config->pipe_src_w,
12259                                &pipe_config->pipe_src_h);
12260
12261 encoder_retry:
12262         /* Ensure the port clock defaults are reset when retrying. */
12263         pipe_config->port_clock = 0;
12264         pipe_config->pixel_multiplier = 1;
12265
12266         /* Fill in default crtc timings, allow encoders to overwrite them. */
12267         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12268                               CRTC_STEREO_DOUBLE);
12269
12270         /* Pass our mode to the connectors and the CRTC to give them a chance to
12271          * adjust it according to limitations or connector properties, and also
12272          * a chance to reject the mode entirely.
12273          */
12274         for_each_connector_in_state(state, connector, connector_state, i) {
12275                 if (connector_state->crtc != crtc)
12276                         continue;
12277
12278                 encoder = to_intel_encoder(connector_state->best_encoder);
12279
12280                 if (!(encoder->compute_config(encoder, pipe_config))) {
12281                         DRM_DEBUG_KMS("Encoder config failure\n");
12282                         goto fail;
12283                 }
12284         }
12285
12286         /* Set default port clock if not overwritten by the encoder. Needs to be
12287          * done afterwards in case the encoder adjusts the mode. */
12288         if (!pipe_config->port_clock)
12289                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12290                         * pipe_config->pixel_multiplier;
12291
12292         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12293         if (ret < 0) {
12294                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12295                 goto fail;
12296         }
12297
12298         if (ret == RETRY) {
12299                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
12300                         ret = -EINVAL;
12301                         goto fail;
12302                 }
12303
12304                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12305                 retry = false;
12306                 goto encoder_retry;
12307         }
12308
12309         /* Dithering seems to not pass-through bits correctly when it should, so
12310          * only enable it on 6bpc panels. */
12311         pipe_config->dither = pipe_config->pipe_bpp == 6*3;
12312         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12313                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12314
12315 fail:
12316         return ret;
12317 }
12318
12319 static void
12320 intel_modeset_update_crtc_state(struct drm_atomic_state *state)
12321 {
12322         struct drm_crtc *crtc;
12323         struct drm_crtc_state *crtc_state;
12324         int i;
12325
12326         /* Double check state. */
12327         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12328                 to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state);
12329
12330                 /* Update hwmode for vblank functions */
12331                 if (crtc->state->active)
12332                         crtc->hwmode = crtc->state->adjusted_mode;
12333                 else
12334                         crtc->hwmode.crtc_clock = 0;
12335         }
12336 }
12337
12338 static bool intel_fuzzy_clock_check(int clock1, int clock2)
12339 {
12340         int diff;
12341
12342         if (clock1 == clock2)
12343                 return true;
12344
12345         if (!clock1 || !clock2)
12346                 return false;
12347
12348         diff = abs(clock1 - clock2);
12349
12350         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12351                 return true;
12352
12353         return false;
12354 }
12355
12356 #define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
12357         list_for_each_entry((intel_crtc), \
12358                             &(dev)->mode_config.crtc_list, \
12359                             base.head) \
12360                 if (mask & (1 <<(intel_crtc)->pipe))
12361
12362 static bool
12363 intel_compare_m_n(unsigned int m, unsigned int n,
12364                   unsigned int m2, unsigned int n2,
12365                   bool exact)
12366 {
12367         if (m == m2 && n == n2)
12368                 return true;
12369
12370         if (exact || !m || !n || !m2 || !n2)
12371                 return false;
12372
12373         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12374
12375         if (m > m2) {
12376                 while (m > m2) {
12377                         m2 <<= 1;
12378                         n2 <<= 1;
12379                 }
12380         } else if (m < m2) {
12381                 while (m < m2) {
12382                         m <<= 1;
12383                         n <<= 1;
12384                 }
12385         }
12386
12387         return m == m2 && n == n2;
12388 }
12389
12390 static bool
12391 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12392                        struct intel_link_m_n *m2_n2,
12393                        bool adjust)
12394 {
12395         if (m_n->tu == m2_n2->tu &&
12396             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12397                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
12398             intel_compare_m_n(m_n->link_m, m_n->link_n,
12399                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
12400                 if (adjust)
12401                         *m2_n2 = *m_n;
12402
12403                 return true;
12404         }
12405
12406         return false;
12407 }
12408
12409 static bool
12410 intel_pipe_config_compare(struct drm_device *dev,
12411                           struct intel_crtc_state *current_config,
12412                           struct intel_crtc_state *pipe_config,
12413                           bool adjust)
12414 {
12415         bool ret = true;
12416
12417 #define INTEL_ERR_OR_DBG_KMS(fmt, ...) \
12418         do { \
12419                 if (!adjust) \
12420                         DRM_ERROR(fmt, ##__VA_ARGS__); \
12421                 else \
12422                         DRM_DEBUG_KMS(fmt, ##__VA_ARGS__); \
12423         } while (0)
12424
12425 #define PIPE_CONF_CHECK_X(name) \
12426         if (current_config->name != pipe_config->name) { \
12427                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12428                           "(expected 0x%08x, found 0x%08x)\n", \
12429                           current_config->name, \
12430                           pipe_config->name); \
12431                 ret = false; \
12432         }
12433
12434 #define PIPE_CONF_CHECK_I(name) \
12435         if (current_config->name != pipe_config->name) { \
12436                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12437                           "(expected %i, found %i)\n", \
12438                           current_config->name, \
12439                           pipe_config->name); \
12440                 ret = false; \
12441         }
12442
12443 #define PIPE_CONF_CHECK_M_N(name) \
12444         if (!intel_compare_link_m_n(&current_config->name, \
12445                                     &pipe_config->name,\
12446                                     adjust)) { \
12447                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12448                           "(expected tu %i gmch %i/%i link %i/%i, " \
12449                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12450                           current_config->name.tu, \
12451                           current_config->name.gmch_m, \
12452                           current_config->name.gmch_n, \
12453                           current_config->name.link_m, \
12454                           current_config->name.link_n, \
12455                           pipe_config->name.tu, \
12456                           pipe_config->name.gmch_m, \
12457                           pipe_config->name.gmch_n, \
12458                           pipe_config->name.link_m, \
12459                           pipe_config->name.link_n); \
12460                 ret = false; \
12461         }
12462
12463 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) \
12464         if (!intel_compare_link_m_n(&current_config->name, \
12465                                     &pipe_config->name, adjust) && \
12466             !intel_compare_link_m_n(&current_config->alt_name, \
12467                                     &pipe_config->name, adjust)) { \
12468                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12469                           "(expected tu %i gmch %i/%i link %i/%i, " \
12470                           "or tu %i gmch %i/%i link %i/%i, " \
12471                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12472                           current_config->name.tu, \
12473                           current_config->name.gmch_m, \
12474                           current_config->name.gmch_n, \
12475                           current_config->name.link_m, \
12476                           current_config->name.link_n, \
12477                           current_config->alt_name.tu, \
12478                           current_config->alt_name.gmch_m, \
12479                           current_config->alt_name.gmch_n, \
12480                           current_config->alt_name.link_m, \
12481                           current_config->alt_name.link_n, \
12482                           pipe_config->name.tu, \
12483                           pipe_config->name.gmch_m, \
12484                           pipe_config->name.gmch_n, \
12485                           pipe_config->name.link_m, \
12486                           pipe_config->name.link_n); \
12487                 ret = false; \
12488         }
12489
12490 /* This is required for BDW+ where there is only one set of registers for
12491  * switching between high and low RR.
12492  * This macro can be used whenever a comparison has to be made between one
12493  * hw state and multiple sw state variables.
12494  */
12495 #define PIPE_CONF_CHECK_I_ALT(name, alt_name) \
12496         if ((current_config->name != pipe_config->name) && \
12497                 (current_config->alt_name != pipe_config->name)) { \
12498                         INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12499                                   "(expected %i or %i, found %i)\n", \
12500                                   current_config->name, \
12501                                   current_config->alt_name, \
12502                                   pipe_config->name); \
12503                         ret = false; \
12504         }
12505
12506 #define PIPE_CONF_CHECK_FLAGS(name, mask)       \
12507         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12508                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name "(" #mask ") " \
12509                           "(expected %i, found %i)\n", \
12510                           current_config->name & (mask), \
12511                           pipe_config->name & (mask)); \
12512                 ret = false; \
12513         }
12514
12515 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \
12516         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12517                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12518                           "(expected %i, found %i)\n", \
12519                           current_config->name, \
12520                           pipe_config->name); \
12521                 ret = false; \
12522         }
12523
12524 #define PIPE_CONF_QUIRK(quirk)  \
12525         ((current_config->quirks | pipe_config->quirks) & (quirk))
12526
12527         PIPE_CONF_CHECK_I(cpu_transcoder);
12528
12529         PIPE_CONF_CHECK_I(has_pch_encoder);
12530         PIPE_CONF_CHECK_I(fdi_lanes);
12531         PIPE_CONF_CHECK_M_N(fdi_m_n);
12532
12533         PIPE_CONF_CHECK_I(has_dp_encoder);
12534         PIPE_CONF_CHECK_I(lane_count);
12535
12536         if (INTEL_INFO(dev)->gen < 8) {
12537                 PIPE_CONF_CHECK_M_N(dp_m_n);
12538
12539                 if (current_config->has_drrs)
12540                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12541         } else
12542                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12543
12544         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12545         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12546         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12547         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12548         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12549         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12550
12551         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12552         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12553         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12554         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12555         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12556         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12557
12558         PIPE_CONF_CHECK_I(pixel_multiplier);
12559         PIPE_CONF_CHECK_I(has_hdmi_sink);
12560         if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
12561             IS_VALLEYVIEW(dev))
12562                 PIPE_CONF_CHECK_I(limited_color_range);
12563         PIPE_CONF_CHECK_I(has_infoframe);
12564
12565         PIPE_CONF_CHECK_I(has_audio);
12566
12567         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12568                               DRM_MODE_FLAG_INTERLACE);
12569
12570         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12571                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12572                                       DRM_MODE_FLAG_PHSYNC);
12573                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12574                                       DRM_MODE_FLAG_NHSYNC);
12575                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12576                                       DRM_MODE_FLAG_PVSYNC);
12577                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12578                                       DRM_MODE_FLAG_NVSYNC);
12579         }
12580
12581         PIPE_CONF_CHECK_X(gmch_pfit.control);
12582         /* pfit ratios are autocomputed by the hw on gen4+ */
12583         if (INTEL_INFO(dev)->gen < 4)
12584                 PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios);
12585         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12586
12587         if (!adjust) {
12588                 PIPE_CONF_CHECK_I(pipe_src_w);
12589                 PIPE_CONF_CHECK_I(pipe_src_h);
12590
12591                 PIPE_CONF_CHECK_I(pch_pfit.enabled);
12592                 if (current_config->pch_pfit.enabled) {
12593                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12594                         PIPE_CONF_CHECK_X(pch_pfit.size);
12595                 }
12596
12597                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12598         }
12599
12600         /* BDW+ don't expose a synchronous way to read the state */
12601         if (IS_HASWELL(dev))
12602                 PIPE_CONF_CHECK_I(ips_enabled);
12603
12604         PIPE_CONF_CHECK_I(double_wide);
12605
12606         PIPE_CONF_CHECK_X(ddi_pll_sel);
12607
12608         PIPE_CONF_CHECK_I(shared_dpll);
12609         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12610         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12611         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12612         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12613         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12614         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12615         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12616         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12617         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12618
12619         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
12620                 PIPE_CONF_CHECK_I(pipe_bpp);
12621
12622         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12623         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12624
12625 #undef PIPE_CONF_CHECK_X
12626 #undef PIPE_CONF_CHECK_I
12627 #undef PIPE_CONF_CHECK_I_ALT
12628 #undef PIPE_CONF_CHECK_FLAGS
12629 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12630 #undef PIPE_CONF_QUIRK
12631 #undef INTEL_ERR_OR_DBG_KMS
12632
12633         return ret;
12634 }
12635
12636 static void check_wm_state(struct drm_device *dev)
12637 {
12638         struct drm_i915_private *dev_priv = dev->dev_private;
12639         struct skl_ddb_allocation hw_ddb, *sw_ddb;
12640         struct intel_crtc *intel_crtc;
12641         int plane;
12642
12643         if (INTEL_INFO(dev)->gen < 9)
12644                 return;
12645
12646         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
12647         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12648
12649         for_each_intel_crtc(dev, intel_crtc) {
12650                 struct skl_ddb_entry *hw_entry, *sw_entry;
12651                 const enum pipe pipe = intel_crtc->pipe;
12652
12653                 if (!intel_crtc->active)
12654                         continue;
12655
12656                 /* planes */
12657                 for_each_plane(dev_priv, pipe, plane) {
12658                         hw_entry = &hw_ddb.plane[pipe][plane];
12659                         sw_entry = &sw_ddb->plane[pipe][plane];
12660
12661                         if (skl_ddb_entry_equal(hw_entry, sw_entry))
12662                                 continue;
12663
12664                         DRM_ERROR("mismatch in DDB state pipe %c plane %d "
12665                                   "(expected (%u,%u), found (%u,%u))\n",
12666                                   pipe_name(pipe), plane + 1,
12667                                   sw_entry->start, sw_entry->end,
12668                                   hw_entry->start, hw_entry->end);
12669                 }
12670
12671                 /* cursor */
12672                 hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
12673                 sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
12674
12675                 if (skl_ddb_entry_equal(hw_entry, sw_entry))
12676                         continue;
12677
12678                 DRM_ERROR("mismatch in DDB state pipe %c cursor "
12679                           "(expected (%u,%u), found (%u,%u))\n",
12680                           pipe_name(pipe),
12681                           sw_entry->start, sw_entry->end,
12682                           hw_entry->start, hw_entry->end);
12683         }
12684 }
12685
12686 static void
12687 check_connector_state(struct drm_device *dev,
12688                       struct drm_atomic_state *old_state)
12689 {
12690         struct drm_connector_state *old_conn_state;
12691         struct drm_connector *connector;
12692         int i;
12693
12694         for_each_connector_in_state(old_state, connector, old_conn_state, i) {
12695                 struct drm_encoder *encoder = connector->encoder;
12696                 struct drm_connector_state *state = connector->state;
12697
12698                 /* This also checks the encoder/connector hw state with the
12699                  * ->get_hw_state callbacks. */
12700                 intel_connector_check_state(to_intel_connector(connector));
12701
12702                 I915_STATE_WARN(state->best_encoder != encoder,
12703                      "connector's atomic encoder doesn't match legacy encoder\n");
12704         }
12705 }
12706
12707 static void
12708 check_encoder_state(struct drm_device *dev)
12709 {
12710         struct intel_encoder *encoder;
12711         struct intel_connector *connector;
12712
12713         for_each_intel_encoder(dev, encoder) {
12714                 bool enabled = false;
12715                 enum pipe pipe;
12716
12717                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
12718                               encoder->base.base.id,
12719                               encoder->base.name);
12720
12721                 for_each_intel_connector(dev, connector) {
12722                         if (connector->base.state->best_encoder != &encoder->base)
12723                                 continue;
12724                         enabled = true;
12725
12726                         I915_STATE_WARN(connector->base.state->crtc !=
12727                                         encoder->base.crtc,
12728                              "connector's crtc doesn't match encoder crtc\n");
12729                 }
12730
12731                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
12732                      "encoder's enabled state mismatch "
12733                      "(expected %i, found %i)\n",
12734                      !!encoder->base.crtc, enabled);
12735
12736                 if (!encoder->base.crtc) {
12737                         bool active;
12738
12739                         active = encoder->get_hw_state(encoder, &pipe);
12740                         I915_STATE_WARN(active,
12741                              "encoder detached but still enabled on pipe %c.\n",
12742                              pipe_name(pipe));
12743                 }
12744         }
12745 }
12746
12747 static void
12748 check_crtc_state(struct drm_device *dev, struct drm_atomic_state *old_state)
12749 {
12750         struct drm_i915_private *dev_priv = dev->dev_private;
12751         struct intel_encoder *encoder;
12752         struct drm_crtc_state *old_crtc_state;
12753         struct drm_crtc *crtc;
12754         int i;
12755
12756         for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
12757                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12758                 struct intel_crtc_state *pipe_config, *sw_config;
12759                 bool active;
12760
12761                 if (!needs_modeset(crtc->state) &&
12762                     !to_intel_crtc_state(crtc->state)->update_pipe)
12763                         continue;
12764
12765                 __drm_atomic_helper_crtc_destroy_state(crtc, old_crtc_state);
12766                 pipe_config = to_intel_crtc_state(old_crtc_state);
12767                 memset(pipe_config, 0, sizeof(*pipe_config));
12768                 pipe_config->base.crtc = crtc;
12769                 pipe_config->base.state = old_state;
12770
12771                 DRM_DEBUG_KMS("[CRTC:%d]\n",
12772                               crtc->base.id);
12773
12774                 active = dev_priv->display.get_pipe_config(intel_crtc,
12775                                                            pipe_config);
12776
12777                 /* hw state is inconsistent with the pipe quirk */
12778                 if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
12779                     (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
12780                         active = crtc->state->active;
12781
12782                 I915_STATE_WARN(crtc->state->active != active,
12783                      "crtc active state doesn't match with hw state "
12784                      "(expected %i, found %i)\n", crtc->state->active, active);
12785
12786                 I915_STATE_WARN(intel_crtc->active != crtc->state->active,
12787                      "transitional active state does not match atomic hw state "
12788                      "(expected %i, found %i)\n", crtc->state->active, intel_crtc->active);
12789
12790                 for_each_encoder_on_crtc(dev, crtc, encoder) {
12791                         enum pipe pipe;
12792
12793                         active = encoder->get_hw_state(encoder, &pipe);
12794                         I915_STATE_WARN(active != crtc->state->active,
12795                                 "[ENCODER:%i] active %i with crtc active %i\n",
12796                                 encoder->base.base.id, active, crtc->state->active);
12797
12798                         I915_STATE_WARN(active && intel_crtc->pipe != pipe,
12799                                         "Encoder connected to wrong pipe %c\n",
12800                                         pipe_name(pipe));
12801
12802                         if (active)
12803                                 encoder->get_config(encoder, pipe_config);
12804                 }
12805
12806                 if (!crtc->state->active)
12807                         continue;
12808
12809                 sw_config = to_intel_crtc_state(crtc->state);
12810                 if (!intel_pipe_config_compare(dev, sw_config,
12811                                                pipe_config, false)) {
12812                         I915_STATE_WARN(1, "pipe state doesn't match!\n");
12813                         intel_dump_pipe_config(intel_crtc, pipe_config,
12814                                                "[hw state]");
12815                         intel_dump_pipe_config(intel_crtc, sw_config,
12816                                                "[sw state]");
12817                 }
12818         }
12819 }
12820
12821 static void
12822 check_shared_dpll_state(struct drm_device *dev)
12823 {
12824         struct drm_i915_private *dev_priv = dev->dev_private;
12825         struct intel_crtc *crtc;
12826         struct intel_dpll_hw_state dpll_hw_state;
12827         int i;
12828
12829         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
12830                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
12831                 int enabled_crtcs = 0, active_crtcs = 0;
12832                 bool active;
12833
12834                 memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12835
12836                 DRM_DEBUG_KMS("%s\n", pll->name);
12837
12838                 active = pll->get_hw_state(dev_priv, pll, &dpll_hw_state);
12839
12840                 I915_STATE_WARN(pll->active > hweight32(pll->config.crtc_mask),
12841                      "more active pll users than references: %i vs %i\n",
12842                      pll->active, hweight32(pll->config.crtc_mask));
12843                 I915_STATE_WARN(pll->active && !pll->on,
12844                      "pll in active use but not on in sw tracking\n");
12845                 I915_STATE_WARN(pll->on && !pll->active,
12846                      "pll in on but not on in use in sw tracking\n");
12847                 I915_STATE_WARN(pll->on != active,
12848                      "pll on state mismatch (expected %i, found %i)\n",
12849                      pll->on, active);
12850
12851                 for_each_intel_crtc(dev, crtc) {
12852                         if (crtc->base.state->enable && intel_crtc_to_shared_dpll(crtc) == pll)
12853                                 enabled_crtcs++;
12854                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll)
12855                                 active_crtcs++;
12856                 }
12857                 I915_STATE_WARN(pll->active != active_crtcs,
12858                      "pll active crtcs mismatch (expected %i, found %i)\n",
12859                      pll->active, active_crtcs);
12860                 I915_STATE_WARN(hweight32(pll->config.crtc_mask) != enabled_crtcs,
12861                      "pll enabled crtcs mismatch (expected %i, found %i)\n",
12862                      hweight32(pll->config.crtc_mask), enabled_crtcs);
12863
12864                 I915_STATE_WARN(pll->on && memcmp(&pll->config.hw_state, &dpll_hw_state,
12865                                        sizeof(dpll_hw_state)),
12866                      "pll hw state mismatch\n");
12867         }
12868 }
12869
12870 static void
12871 intel_modeset_check_state(struct drm_device *dev,
12872                           struct drm_atomic_state *old_state)
12873 {
12874         check_wm_state(dev);
12875         check_connector_state(dev, old_state);
12876         check_encoder_state(dev);
12877         check_crtc_state(dev, old_state);
12878         check_shared_dpll_state(dev);
12879 }
12880
12881 void ironlake_check_encoder_dotclock(const struct intel_crtc_state *pipe_config,
12882                                      int dotclock)
12883 {
12884         /*
12885          * FDI already provided one idea for the dotclock.
12886          * Yell if the encoder disagrees.
12887          */
12888         WARN(!intel_fuzzy_clock_check(pipe_config->base.adjusted_mode.crtc_clock, dotclock),
12889              "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12890              pipe_config->base.adjusted_mode.crtc_clock, dotclock);
12891 }
12892
12893 static void update_scanline_offset(struct intel_crtc *crtc)
12894 {
12895         struct drm_device *dev = crtc->base.dev;
12896
12897         /*
12898          * The scanline counter increments at the leading edge of hsync.
12899          *
12900          * On most platforms it starts counting from vtotal-1 on the
12901          * first active line. That means the scanline counter value is
12902          * always one less than what we would expect. Ie. just after
12903          * start of vblank, which also occurs at start of hsync (on the
12904          * last active line), the scanline counter will read vblank_start-1.
12905          *
12906          * On gen2 the scanline counter starts counting from 1 instead
12907          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12908          * to keep the value positive), instead of adding one.
12909          *
12910          * On HSW+ the behaviour of the scanline counter depends on the output
12911          * type. For DP ports it behaves like most other platforms, but on HDMI
12912          * there's an extra 1 line difference. So we need to add two instead of
12913          * one to the value.
12914          */
12915         if (IS_GEN2(dev)) {
12916                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
12917                 int vtotal;
12918
12919                 vtotal = adjusted_mode->crtc_vtotal;
12920                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12921                         vtotal /= 2;
12922
12923                 crtc->scanline_offset = vtotal - 1;
12924         } else if (HAS_DDI(dev) &&
12925                    intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
12926                 crtc->scanline_offset = 2;
12927         } else
12928                 crtc->scanline_offset = 1;
12929 }
12930
12931 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12932 {
12933         struct drm_device *dev = state->dev;
12934         struct drm_i915_private *dev_priv = to_i915(dev);
12935         struct intel_shared_dpll_config *shared_dpll = NULL;
12936         struct intel_crtc *intel_crtc;
12937         struct intel_crtc_state *intel_crtc_state;
12938         struct drm_crtc *crtc;
12939         struct drm_crtc_state *crtc_state;
12940         int i;
12941
12942         if (!dev_priv->display.crtc_compute_clock)
12943                 return;
12944
12945         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12946                 int dpll;
12947
12948                 intel_crtc = to_intel_crtc(crtc);
12949                 intel_crtc_state = to_intel_crtc_state(crtc_state);
12950                 dpll = intel_crtc_state->shared_dpll;
12951
12952                 if (!needs_modeset(crtc_state) || dpll == DPLL_ID_PRIVATE)
12953                         continue;
12954
12955                 intel_crtc_state->shared_dpll = DPLL_ID_PRIVATE;
12956
12957                 if (!shared_dpll)
12958                         shared_dpll = intel_atomic_get_shared_dpll_state(state);
12959
12960                 shared_dpll[dpll].crtc_mask &= ~(1 << intel_crtc->pipe);
12961         }
12962 }
12963
12964 /*
12965  * This implements the workaround described in the "notes" section of the mode
12966  * set sequence documentation. When going from no pipes or single pipe to
12967  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12968  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12969  */
12970 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12971 {
12972         struct drm_crtc_state *crtc_state;
12973         struct intel_crtc *intel_crtc;
12974         struct drm_crtc *crtc;
12975         struct intel_crtc_state *first_crtc_state = NULL;
12976         struct intel_crtc_state *other_crtc_state = NULL;
12977         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12978         int i;
12979
12980         /* look at all crtc's that are going to be enabled in during modeset */
12981         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12982                 intel_crtc = to_intel_crtc(crtc);
12983
12984                 if (!crtc_state->active || !needs_modeset(crtc_state))
12985                         continue;
12986
12987                 if (first_crtc_state) {
12988                         other_crtc_state = to_intel_crtc_state(crtc_state);
12989                         break;
12990                 } else {
12991                         first_crtc_state = to_intel_crtc_state(crtc_state);
12992                         first_pipe = intel_crtc->pipe;
12993                 }
12994         }
12995
12996         /* No workaround needed? */
12997         if (!first_crtc_state)
12998                 return 0;
12999
13000         /* w/a possibly needed, check how many crtc's are already enabled. */
13001         for_each_intel_crtc(state->dev, intel_crtc) {
13002                 struct intel_crtc_state *pipe_config;
13003
13004                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
13005                 if (IS_ERR(pipe_config))
13006                         return PTR_ERR(pipe_config);
13007
13008                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
13009
13010                 if (!pipe_config->base.active ||
13011                     needs_modeset(&pipe_config->base))
13012                         continue;
13013
13014                 /* 2 or more enabled crtcs means no need for w/a */
13015                 if (enabled_pipe != INVALID_PIPE)
13016                         return 0;
13017
13018                 enabled_pipe = intel_crtc->pipe;
13019         }
13020
13021         if (enabled_pipe != INVALID_PIPE)
13022                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13023         else if (other_crtc_state)
13024                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13025
13026         return 0;
13027 }
13028
13029 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
13030 {
13031         struct drm_crtc *crtc;
13032         struct drm_crtc_state *crtc_state;
13033         int ret = 0;
13034
13035         /* add all active pipes to the state */
13036         for_each_crtc(state->dev, crtc) {
13037                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13038                 if (IS_ERR(crtc_state))
13039                         return PTR_ERR(crtc_state);
13040
13041                 if (!crtc_state->active || needs_modeset(crtc_state))
13042                         continue;
13043
13044                 crtc_state->mode_changed = true;
13045
13046                 ret = drm_atomic_add_affected_connectors(state, crtc);
13047                 if (ret)
13048                         break;
13049
13050                 ret = drm_atomic_add_affected_planes(state, crtc);
13051                 if (ret)
13052                         break;
13053         }
13054
13055         return ret;
13056 }
13057
13058 static int intel_modeset_checks(struct drm_atomic_state *state)
13059 {
13060         struct drm_device *dev = state->dev;
13061         struct drm_i915_private *dev_priv = dev->dev_private;
13062         int ret;
13063
13064         if (!check_digital_port_conflicts(state)) {
13065                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13066                 return -EINVAL;
13067         }
13068
13069         /*
13070          * See if the config requires any additional preparation, e.g.
13071          * to adjust global state with pipes off.  We need to do this
13072          * here so we can get the modeset_pipe updated config for the new
13073          * mode set on this crtc.  For other crtcs we need to use the
13074          * adjusted_mode bits in the crtc directly.
13075          */
13076         if (dev_priv->display.modeset_calc_cdclk) {
13077                 unsigned int cdclk;
13078
13079                 ret = dev_priv->display.modeset_calc_cdclk(state);
13080
13081                 cdclk = to_intel_atomic_state(state)->cdclk;
13082                 if (!ret && cdclk != dev_priv->cdclk_freq)
13083                         ret = intel_modeset_all_pipes(state);
13084
13085                 if (ret < 0)
13086                         return ret;
13087         } else
13088                 to_intel_atomic_state(state)->cdclk = dev_priv->cdclk_freq;
13089
13090         intel_modeset_clear_plls(state);
13091
13092         if (IS_HASWELL(dev))
13093                 return haswell_mode_set_planes_workaround(state);
13094
13095         return 0;
13096 }
13097
13098 /**
13099  * intel_atomic_check - validate state object
13100  * @dev: drm device
13101  * @state: state to validate
13102  */
13103 static int intel_atomic_check(struct drm_device *dev,
13104                               struct drm_atomic_state *state)
13105 {
13106         struct drm_crtc *crtc;
13107         struct drm_crtc_state *crtc_state;
13108         int ret, i;
13109         bool any_ms = false;
13110
13111         ret = drm_atomic_helper_check_modeset(dev, state);
13112         if (ret)
13113                 return ret;
13114
13115         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13116                 struct intel_crtc_state *pipe_config =
13117                         to_intel_crtc_state(crtc_state);
13118
13119                 memset(&to_intel_crtc(crtc)->atomic, 0,
13120                        sizeof(struct intel_crtc_atomic_commit));
13121
13122                 /* Catch I915_MODE_FLAG_INHERITED */
13123                 if (crtc_state->mode.private_flags != crtc->state->mode.private_flags)
13124                         crtc_state->mode_changed = true;
13125
13126                 if (!crtc_state->enable) {
13127                         if (needs_modeset(crtc_state))
13128                                 any_ms = true;
13129                         continue;
13130                 }
13131
13132                 if (!needs_modeset(crtc_state))
13133                         continue;
13134
13135                 /* FIXME: For only active_changed we shouldn't need to do any
13136                  * state recomputation at all. */
13137
13138                 ret = drm_atomic_add_affected_connectors(state, crtc);
13139                 if (ret)
13140                         return ret;
13141
13142                 ret = intel_modeset_pipe_config(crtc, pipe_config);
13143                 if (ret)
13144                         return ret;
13145
13146                 if (i915.fastboot &&
13147                     intel_pipe_config_compare(state->dev,
13148                                         to_intel_crtc_state(crtc->state),
13149                                         pipe_config, true)) {
13150                         crtc_state->mode_changed = false;
13151                         to_intel_crtc_state(crtc_state)->update_pipe = true;
13152                 }
13153
13154                 if (needs_modeset(crtc_state)) {
13155                         any_ms = true;
13156
13157                         ret = drm_atomic_add_affected_planes(state, crtc);
13158                         if (ret)
13159                                 return ret;
13160                 }
13161
13162                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
13163                                        needs_modeset(crtc_state) ?
13164                                        "[modeset]" : "[fastset]");
13165         }
13166
13167         if (any_ms) {
13168                 ret = intel_modeset_checks(state);
13169
13170                 if (ret)
13171                         return ret;
13172         } else
13173                 to_intel_atomic_state(state)->cdclk =
13174                         to_i915(state->dev)->cdclk_freq;
13175
13176         return drm_atomic_helper_check_planes(state->dev, state);
13177 }
13178
13179 /**
13180  * intel_atomic_commit - commit validated state object
13181  * @dev: DRM device
13182  * @state: the top-level driver state object
13183  * @async: asynchronous commit
13184  *
13185  * This function commits a top-level state object that has been validated
13186  * with drm_atomic_helper_check().
13187  *
13188  * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
13189  * we can only handle plane-related operations and do not yet support
13190  * asynchronous commit.
13191  *
13192  * RETURNS
13193  * Zero for success or -errno.
13194  */
13195 static int intel_atomic_commit(struct drm_device *dev,
13196                                struct drm_atomic_state *state,
13197                                bool async)
13198 {
13199         struct drm_i915_private *dev_priv = dev->dev_private;
13200         struct drm_crtc *crtc;
13201         struct drm_crtc_state *crtc_state;
13202         int ret = 0;
13203         int i;
13204         bool any_ms = false;
13205
13206         if (async) {
13207                 DRM_DEBUG_KMS("i915 does not yet support async commit\n");
13208                 return -EINVAL;
13209         }
13210
13211         ret = drm_atomic_helper_prepare_planes(dev, state);
13212         if (ret)
13213                 return ret;
13214
13215         drm_atomic_helper_swap_state(dev, state);
13216
13217         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13218                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13219
13220                 if (!needs_modeset(crtc->state))
13221                         continue;
13222
13223                 any_ms = true;
13224                 intel_pre_plane_update(intel_crtc);
13225
13226                 if (crtc_state->active) {
13227                         intel_crtc_disable_planes(crtc, crtc_state->plane_mask);
13228                         dev_priv->display.crtc_disable(crtc);
13229                         intel_crtc->active = false;
13230                         intel_disable_shared_dpll(intel_crtc);
13231                 }
13232         }
13233
13234         /* Only after disabling all output pipelines that will be changed can we
13235          * update the the output configuration. */
13236         intel_modeset_update_crtc_state(state);
13237
13238         if (any_ms) {
13239                 intel_shared_dpll_commit(state);
13240
13241                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
13242                 modeset_update_crtc_power_domains(state);
13243         }
13244
13245         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13246         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13247                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13248                 bool modeset = needs_modeset(crtc->state);
13249                 bool update_pipe = !modeset &&
13250                         to_intel_crtc_state(crtc->state)->update_pipe;
13251                 unsigned long put_domains = 0;
13252
13253                 if (modeset && crtc->state->active) {
13254                         update_scanline_offset(to_intel_crtc(crtc));
13255                         dev_priv->display.crtc_enable(crtc);
13256                 }
13257
13258                 if (update_pipe) {
13259                         put_domains = modeset_get_crtc_power_domains(crtc);
13260
13261                         /* make sure intel_modeset_check_state runs */
13262                         any_ms = true;
13263                 }
13264
13265                 if (!modeset)
13266                         intel_pre_plane_update(intel_crtc);
13267
13268                 drm_atomic_helper_commit_planes_on_crtc(crtc_state);
13269
13270                 if (put_domains)
13271                         modeset_put_power_domains(dev_priv, put_domains);
13272
13273                 intel_post_plane_update(intel_crtc);
13274         }
13275
13276         /* FIXME: add subpixel order */
13277
13278         drm_atomic_helper_wait_for_vblanks(dev, state);
13279         drm_atomic_helper_cleanup_planes(dev, state);
13280
13281         if (any_ms)
13282                 intel_modeset_check_state(dev, state);
13283
13284         drm_atomic_state_free(state);
13285
13286         return 0;
13287 }
13288
13289 void intel_crtc_restore_mode(struct drm_crtc *crtc)
13290 {
13291         struct drm_device *dev = crtc->dev;
13292         struct drm_atomic_state *state;
13293         struct drm_crtc_state *crtc_state;
13294         int ret;
13295
13296         state = drm_atomic_state_alloc(dev);
13297         if (!state) {
13298                 DRM_DEBUG_KMS("[CRTC:%d] crtc restore failed, out of memory",
13299                               crtc->base.id);
13300                 return;
13301         }
13302
13303         state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
13304
13305 retry:
13306         crtc_state = drm_atomic_get_crtc_state(state, crtc);
13307         ret = PTR_ERR_OR_ZERO(crtc_state);
13308         if (!ret) {
13309                 if (!crtc_state->active)
13310                         goto out;
13311
13312                 crtc_state->mode_changed = true;
13313                 ret = drm_atomic_commit(state);
13314         }
13315
13316         if (ret == -EDEADLK) {
13317                 drm_atomic_state_clear(state);
13318                 drm_modeset_backoff(state->acquire_ctx);
13319                 goto retry;
13320         }
13321
13322         if (ret)
13323 out:
13324                 drm_atomic_state_free(state);
13325 }
13326
13327 #undef for_each_intel_crtc_masked
13328
13329 static const struct drm_crtc_funcs intel_crtc_funcs = {
13330         .gamma_set = intel_crtc_gamma_set,
13331         .set_config = drm_atomic_helper_set_config,
13332         .destroy = intel_crtc_destroy,
13333         .page_flip = intel_crtc_page_flip,
13334         .atomic_duplicate_state = intel_crtc_duplicate_state,
13335         .atomic_destroy_state = intel_crtc_destroy_state,
13336 };
13337
13338 static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
13339                                       struct intel_shared_dpll *pll,
13340                                       struct intel_dpll_hw_state *hw_state)
13341 {
13342         uint32_t val;
13343
13344         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PLLS))
13345                 return false;
13346
13347         val = I915_READ(PCH_DPLL(pll->id));
13348         hw_state->dpll = val;
13349         hw_state->fp0 = I915_READ(PCH_FP0(pll->id));
13350         hw_state->fp1 = I915_READ(PCH_FP1(pll->id));
13351
13352         return val & DPLL_VCO_ENABLE;
13353 }
13354
13355 static void ibx_pch_dpll_mode_set(struct drm_i915_private *dev_priv,
13356                                   struct intel_shared_dpll *pll)
13357 {
13358         I915_WRITE(PCH_FP0(pll->id), pll->config.hw_state.fp0);
13359         I915_WRITE(PCH_FP1(pll->id), pll->config.hw_state.fp1);
13360 }
13361
13362 static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
13363                                 struct intel_shared_dpll *pll)
13364 {
13365         /* PCH refclock must be enabled first */
13366         ibx_assert_pch_refclk_enabled(dev_priv);
13367
13368         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13369
13370         /* Wait for the clocks to stabilize. */
13371         POSTING_READ(PCH_DPLL(pll->id));
13372         udelay(150);
13373
13374         /* The pixel multiplier can only be updated once the
13375          * DPLL is enabled and the clocks are stable.
13376          *
13377          * So write it again.
13378          */
13379         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13380         POSTING_READ(PCH_DPLL(pll->id));
13381         udelay(200);
13382 }
13383
13384 static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
13385                                  struct intel_shared_dpll *pll)
13386 {
13387         struct drm_device *dev = dev_priv->dev;
13388         struct intel_crtc *crtc;
13389
13390         /* Make sure no transcoder isn't still depending on us. */
13391         for_each_intel_crtc(dev, crtc) {
13392                 if (intel_crtc_to_shared_dpll(crtc) == pll)
13393                         assert_pch_transcoder_disabled(dev_priv, crtc->pipe);
13394         }
13395
13396         I915_WRITE(PCH_DPLL(pll->id), 0);
13397         POSTING_READ(PCH_DPLL(pll->id));
13398         udelay(200);
13399 }
13400
13401 static char *ibx_pch_dpll_names[] = {
13402         "PCH DPLL A",
13403         "PCH DPLL B",
13404 };
13405
13406 static void ibx_pch_dpll_init(struct drm_device *dev)
13407 {
13408         struct drm_i915_private *dev_priv = dev->dev_private;
13409         int i;
13410
13411         dev_priv->num_shared_dpll = 2;
13412
13413         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
13414                 dev_priv->shared_dplls[i].id = i;
13415                 dev_priv->shared_dplls[i].name = ibx_pch_dpll_names[i];
13416                 dev_priv->shared_dplls[i].mode_set = ibx_pch_dpll_mode_set;
13417                 dev_priv->shared_dplls[i].enable = ibx_pch_dpll_enable;
13418                 dev_priv->shared_dplls[i].disable = ibx_pch_dpll_disable;
13419                 dev_priv->shared_dplls[i].get_hw_state =
13420                         ibx_pch_dpll_get_hw_state;
13421         }
13422 }
13423
13424 static void intel_shared_dpll_init(struct drm_device *dev)
13425 {
13426         struct drm_i915_private *dev_priv = dev->dev_private;
13427
13428         if (HAS_DDI(dev))
13429                 intel_ddi_pll_init(dev);
13430         else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
13431                 ibx_pch_dpll_init(dev);
13432         else
13433                 dev_priv->num_shared_dpll = 0;
13434
13435         BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
13436 }
13437
13438 /**
13439  * intel_prepare_plane_fb - Prepare fb for usage on plane
13440  * @plane: drm plane to prepare for
13441  * @fb: framebuffer to prepare for presentation
13442  *
13443  * Prepares a framebuffer for usage on a display plane.  Generally this
13444  * involves pinning the underlying object and updating the frontbuffer tracking
13445  * bits.  Some older platforms need special physical address handling for
13446  * cursor planes.
13447  *
13448  * Returns 0 on success, negative error code on failure.
13449  */
13450 int
13451 intel_prepare_plane_fb(struct drm_plane *plane,
13452                        const struct drm_plane_state *new_state)
13453 {
13454         struct drm_device *dev = plane->dev;
13455         struct drm_framebuffer *fb = new_state->fb;
13456         struct intel_plane *intel_plane = to_intel_plane(plane);
13457         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13458         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
13459         int ret = 0;
13460
13461         if (!obj)
13462                 return 0;
13463
13464         mutex_lock(&dev->struct_mutex);
13465
13466         if (plane->type == DRM_PLANE_TYPE_CURSOR &&
13467             INTEL_INFO(dev)->cursor_needs_physical) {
13468                 int align = IS_I830(dev) ? 16 * 1024 : 256;
13469                 ret = i915_gem_object_attach_phys(obj, align);
13470                 if (ret)
13471                         DRM_DEBUG_KMS("failed to attach phys object\n");
13472         } else {
13473                 ret = intel_pin_and_fence_fb_obj(plane, fb, new_state, NULL, NULL);
13474         }
13475
13476         if (ret == 0)
13477                 i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
13478
13479         mutex_unlock(&dev->struct_mutex);
13480
13481         return ret;
13482 }
13483
13484 /**
13485  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13486  * @plane: drm plane to clean up for
13487  * @fb: old framebuffer that was on plane
13488  *
13489  * Cleans up a framebuffer that has just been removed from a plane.
13490  */
13491 void
13492 intel_cleanup_plane_fb(struct drm_plane *plane,
13493                        const struct drm_plane_state *old_state)
13494 {
13495         struct drm_device *dev = plane->dev;
13496         struct drm_i915_gem_object *obj = intel_fb_obj(old_state->fb);
13497
13498         if (!obj)
13499                 return;
13500
13501         if (plane->type != DRM_PLANE_TYPE_CURSOR ||
13502             !INTEL_INFO(dev)->cursor_needs_physical) {
13503                 mutex_lock(&dev->struct_mutex);
13504                 intel_unpin_fb_obj(old_state->fb, old_state);
13505                 mutex_unlock(&dev->struct_mutex);
13506         }
13507 }
13508
13509 int
13510 skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
13511 {
13512         int max_scale;
13513         struct drm_device *dev;
13514         struct drm_i915_private *dev_priv;
13515         int crtc_clock, cdclk;
13516
13517         if (!intel_crtc || !crtc_state)
13518                 return DRM_PLANE_HELPER_NO_SCALING;
13519
13520         dev = intel_crtc->base.dev;
13521         dev_priv = dev->dev_private;
13522         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13523         cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
13524
13525         if (!crtc_clock || !cdclk)
13526                 return DRM_PLANE_HELPER_NO_SCALING;
13527
13528         /*
13529          * skl max scale is lower of:
13530          *    close to 3 but not 3, -1 is for that purpose
13531          *            or
13532          *    cdclk/crtc_clock
13533          */
13534         max_scale = min((1 << 16) * 3 - 1, (1 << 8) * ((cdclk << 8) / crtc_clock));
13535
13536         return max_scale;
13537 }
13538
13539 static int
13540 intel_check_primary_plane(struct drm_plane *plane,
13541                           struct intel_crtc_state *crtc_state,
13542                           struct intel_plane_state *state)
13543 {
13544         struct drm_crtc *crtc = state->base.crtc;
13545         struct drm_framebuffer *fb = state->base.fb;
13546         int min_scale = DRM_PLANE_HELPER_NO_SCALING;
13547         int max_scale = DRM_PLANE_HELPER_NO_SCALING;
13548         bool can_position = false;
13549
13550         if (INTEL_INFO(plane->dev)->gen >= 9) {
13551                 /* use scaler when colorkey is not required */
13552                 if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
13553                         min_scale = 1;
13554                         max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state);
13555                 }
13556                 can_position = true;
13557         }
13558
13559         return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13560                                              &state->dst, &state->clip,
13561                                              min_scale, max_scale,
13562                                              can_position, true,
13563                                              &state->visible);
13564 }
13565
13566 static void
13567 intel_commit_primary_plane(struct drm_plane *plane,
13568                            struct intel_plane_state *state)
13569 {
13570         struct drm_crtc *crtc = state->base.crtc;
13571         struct drm_framebuffer *fb = state->base.fb;
13572         struct drm_device *dev = plane->dev;
13573         struct drm_i915_private *dev_priv = dev->dev_private;
13574         struct intel_crtc *intel_crtc;
13575         struct drm_rect *src = &state->src;
13576
13577         crtc = crtc ? crtc : plane->crtc;
13578         intel_crtc = to_intel_crtc(crtc);
13579
13580         plane->fb = fb;
13581         crtc->x = src->x1 >> 16;
13582         crtc->y = src->y1 >> 16;
13583
13584         if (!crtc->state->active)
13585                 return;
13586
13587         dev_priv->display.update_primary_plane(crtc, fb,
13588                                                state->src.x1 >> 16,
13589                                                state->src.y1 >> 16);
13590 }
13591
13592 static void
13593 intel_disable_primary_plane(struct drm_plane *plane,
13594                             struct drm_crtc *crtc)
13595 {
13596         struct drm_device *dev = plane->dev;
13597         struct drm_i915_private *dev_priv = dev->dev_private;
13598
13599         dev_priv->display.update_primary_plane(crtc, NULL, 0, 0);
13600 }
13601
13602 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13603                                     struct drm_crtc_state *old_crtc_state)
13604 {
13605         struct drm_device *dev = crtc->dev;
13606         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13607         struct intel_crtc_state *old_intel_state =
13608                 to_intel_crtc_state(old_crtc_state);
13609         bool modeset = needs_modeset(crtc->state);
13610
13611         if (intel_crtc->atomic.update_wm_pre)
13612                 intel_update_watermarks(crtc);
13613
13614         /* Perform vblank evasion around commit operation */
13615         if (crtc->state->active)
13616                 intel_pipe_update_start(intel_crtc);
13617
13618         if (modeset)
13619                 return;
13620
13621         if (to_intel_crtc_state(crtc->state)->update_pipe)
13622                 intel_update_pipe_config(intel_crtc, old_intel_state);
13623         else if (INTEL_INFO(dev)->gen >= 9)
13624                 skl_detach_scalers(intel_crtc);
13625 }
13626
13627 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13628                                      struct drm_crtc_state *old_crtc_state)
13629 {
13630         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13631
13632         if (crtc->state->active)
13633                 intel_pipe_update_end(intel_crtc);
13634 }
13635
13636 /**
13637  * intel_plane_destroy - destroy a plane
13638  * @plane: plane to destroy
13639  *
13640  * Common destruction function for all types of planes (primary, cursor,
13641  * sprite).
13642  */
13643 void intel_plane_destroy(struct drm_plane *plane)
13644 {
13645         struct intel_plane *intel_plane = to_intel_plane(plane);
13646         drm_plane_cleanup(plane);
13647         kfree(intel_plane);
13648 }
13649
13650 const struct drm_plane_funcs intel_plane_funcs = {
13651         .update_plane = drm_atomic_helper_update_plane,
13652         .disable_plane = drm_atomic_helper_disable_plane,
13653         .destroy = intel_plane_destroy,
13654         .set_property = drm_atomic_helper_plane_set_property,
13655         .atomic_get_property = intel_plane_atomic_get_property,
13656         .atomic_set_property = intel_plane_atomic_set_property,
13657         .atomic_duplicate_state = intel_plane_duplicate_state,
13658         .atomic_destroy_state = intel_plane_destroy_state,
13659
13660 };
13661
13662 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
13663                                                     int pipe)
13664 {
13665         struct intel_plane *primary;
13666         struct intel_plane_state *state;
13667         const uint32_t *intel_primary_formats;
13668         unsigned int num_formats;
13669
13670         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
13671         if (primary == NULL)
13672                 return NULL;
13673
13674         state = intel_create_plane_state(&primary->base);
13675         if (!state) {
13676                 kfree(primary);
13677                 return NULL;
13678         }
13679         primary->base.state = &state->base;
13680
13681         primary->can_scale = false;
13682         primary->max_downscale = 1;
13683         if (INTEL_INFO(dev)->gen >= 9) {
13684                 primary->can_scale = true;
13685                 state->scaler_id = -1;
13686         }
13687         primary->pipe = pipe;
13688         primary->plane = pipe;
13689         primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
13690         primary->check_plane = intel_check_primary_plane;
13691         primary->commit_plane = intel_commit_primary_plane;
13692         primary->disable_plane = intel_disable_primary_plane;
13693         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
13694                 primary->plane = !pipe;
13695
13696         if (INTEL_INFO(dev)->gen >= 9) {
13697                 intel_primary_formats = skl_primary_formats;
13698                 num_formats = ARRAY_SIZE(skl_primary_formats);
13699         } else if (INTEL_INFO(dev)->gen >= 4) {
13700                 intel_primary_formats = i965_primary_formats;
13701                 num_formats = ARRAY_SIZE(i965_primary_formats);
13702         } else {
13703                 intel_primary_formats = i8xx_primary_formats;
13704                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13705         }
13706
13707         drm_universal_plane_init(dev, &primary->base, 0,
13708                                  &intel_plane_funcs,
13709                                  intel_primary_formats, num_formats,
13710                                  DRM_PLANE_TYPE_PRIMARY);
13711
13712         if (INTEL_INFO(dev)->gen >= 4)
13713                 intel_create_rotation_property(dev, primary);
13714
13715         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
13716
13717         return &primary->base;
13718 }
13719
13720 void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
13721 {
13722         if (!dev->mode_config.rotation_property) {
13723                 unsigned long flags = BIT(DRM_ROTATE_0) |
13724                         BIT(DRM_ROTATE_180);
13725
13726                 if (INTEL_INFO(dev)->gen >= 9)
13727                         flags |= BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270);
13728
13729                 dev->mode_config.rotation_property =
13730                         drm_mode_create_rotation_property(dev, flags);
13731         }
13732         if (dev->mode_config.rotation_property)
13733                 drm_object_attach_property(&plane->base.base,
13734                                 dev->mode_config.rotation_property,
13735                                 plane->base.state->rotation);
13736 }
13737
13738 static int
13739 intel_check_cursor_plane(struct drm_plane *plane,
13740                          struct intel_crtc_state *crtc_state,
13741                          struct intel_plane_state *state)
13742 {
13743         struct drm_crtc *crtc = crtc_state->base.crtc;
13744         struct drm_framebuffer *fb = state->base.fb;
13745         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13746         enum pipe pipe = to_intel_plane(plane)->pipe;
13747         unsigned stride;
13748         int ret;
13749
13750         ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13751                                             &state->dst, &state->clip,
13752                                             DRM_PLANE_HELPER_NO_SCALING,
13753                                             DRM_PLANE_HELPER_NO_SCALING,
13754                                             true, true, &state->visible);
13755         if (ret)
13756                 return ret;
13757
13758         /* if we want to turn off the cursor ignore width and height */
13759         if (!obj)
13760                 return 0;
13761
13762         /* Check for which cursor types we support */
13763         if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
13764                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
13765                           state->base.crtc_w, state->base.crtc_h);
13766                 return -EINVAL;
13767         }
13768
13769         stride = roundup_pow_of_two(state->base.crtc_w) * 4;
13770         if (obj->base.size < stride * state->base.crtc_h) {
13771                 DRM_DEBUG_KMS("buffer is too small\n");
13772                 return -ENOMEM;
13773         }
13774
13775         if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) {
13776                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
13777                 return -EINVAL;
13778         }
13779
13780         /*
13781          * There's something wrong with the cursor on CHV pipe C.
13782          * If it straddles the left edge of the screen then
13783          * moving it away from the edge or disabling it often
13784          * results in a pipe underrun, and often that can lead to
13785          * dead pipe (constant underrun reported, and it scans
13786          * out just a solid color). To recover from that, the
13787          * display power well must be turned off and on again.
13788          * Refuse the put the cursor into that compromised position.
13789          */
13790         if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
13791             state->visible && state->base.crtc_x < 0) {
13792                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
13793                 return -EINVAL;
13794         }
13795
13796         return 0;
13797 }
13798
13799 static void
13800 intel_disable_cursor_plane(struct drm_plane *plane,
13801                            struct drm_crtc *crtc)
13802 {
13803         intel_crtc_update_cursor(crtc, false);
13804 }
13805
13806 static void
13807 intel_commit_cursor_plane(struct drm_plane *plane,
13808                           struct intel_plane_state *state)
13809 {
13810         struct drm_crtc *crtc = state->base.crtc;
13811         struct drm_device *dev = plane->dev;
13812         struct intel_crtc *intel_crtc;
13813         struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
13814         uint32_t addr;
13815
13816         crtc = crtc ? crtc : plane->crtc;
13817         intel_crtc = to_intel_crtc(crtc);
13818
13819         if (!obj)
13820                 addr = 0;
13821         else if (!INTEL_INFO(dev)->cursor_needs_physical)
13822                 addr = i915_gem_obj_ggtt_offset(obj);
13823         else
13824                 addr = obj->phys_handle->busaddr;
13825
13826         intel_crtc->cursor_addr = addr;
13827
13828         if (crtc->state->active)
13829                 intel_crtc_update_cursor(crtc, state->visible);
13830 }
13831
13832 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
13833                                                    int pipe)
13834 {
13835         struct intel_plane *cursor;
13836         struct intel_plane_state *state;
13837
13838         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
13839         if (cursor == NULL)
13840                 return NULL;
13841
13842         state = intel_create_plane_state(&cursor->base);
13843         if (!state) {
13844                 kfree(cursor);
13845                 return NULL;
13846         }
13847         cursor->base.state = &state->base;
13848
13849         cursor->can_scale = false;
13850         cursor->max_downscale = 1;
13851         cursor->pipe = pipe;
13852         cursor->plane = pipe;
13853         cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
13854         cursor->check_plane = intel_check_cursor_plane;
13855         cursor->commit_plane = intel_commit_cursor_plane;
13856         cursor->disable_plane = intel_disable_cursor_plane;
13857
13858         drm_universal_plane_init(dev, &cursor->base, 0,
13859                                  &intel_plane_funcs,
13860                                  intel_cursor_formats,
13861                                  ARRAY_SIZE(intel_cursor_formats),
13862                                  DRM_PLANE_TYPE_CURSOR);
13863
13864         if (INTEL_INFO(dev)->gen >= 4) {
13865                 if (!dev->mode_config.rotation_property)
13866                         dev->mode_config.rotation_property =
13867                                 drm_mode_create_rotation_property(dev,
13868                                                         BIT(DRM_ROTATE_0) |
13869                                                         BIT(DRM_ROTATE_180));
13870                 if (dev->mode_config.rotation_property)
13871                         drm_object_attach_property(&cursor->base.base,
13872                                 dev->mode_config.rotation_property,
13873                                 state->base.rotation);
13874         }
13875
13876         if (INTEL_INFO(dev)->gen >=9)
13877                 state->scaler_id = -1;
13878
13879         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
13880
13881         return &cursor->base;
13882 }
13883
13884 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
13885         struct intel_crtc_state *crtc_state)
13886 {
13887         int i;
13888         struct intel_scaler *intel_scaler;
13889         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
13890
13891         for (i = 0; i < intel_crtc->num_scalers; i++) {
13892                 intel_scaler = &scaler_state->scalers[i];
13893                 intel_scaler->in_use = 0;
13894                 intel_scaler->mode = PS_SCALER_MODE_DYN;
13895         }
13896
13897         scaler_state->scaler_id = -1;
13898 }
13899
13900 static void intel_crtc_init(struct drm_device *dev, int pipe)
13901 {
13902         struct drm_i915_private *dev_priv = dev->dev_private;
13903         struct intel_crtc *intel_crtc;
13904         struct intel_crtc_state *crtc_state = NULL;
13905         struct drm_plane *primary = NULL;
13906         struct drm_plane *cursor = NULL;
13907         int i, ret;
13908
13909         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
13910         if (intel_crtc == NULL)
13911                 return;
13912
13913         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
13914         if (!crtc_state)
13915                 goto fail;
13916         intel_crtc->config = crtc_state;
13917         intel_crtc->base.state = &crtc_state->base;
13918         crtc_state->base.crtc = &intel_crtc->base;
13919
13920         /* initialize shared scalers */
13921         if (INTEL_INFO(dev)->gen >= 9) {
13922                 if (pipe == PIPE_C)
13923                         intel_crtc->num_scalers = 1;
13924                 else
13925                         intel_crtc->num_scalers = SKL_NUM_SCALERS;
13926
13927                 skl_init_scalers(dev, intel_crtc, crtc_state);
13928         }
13929
13930         primary = intel_primary_plane_create(dev, pipe);
13931         if (!primary)
13932                 goto fail;
13933
13934         cursor = intel_cursor_plane_create(dev, pipe);
13935         if (!cursor)
13936                 goto fail;
13937
13938         ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
13939                                         cursor, &intel_crtc_funcs);
13940         if (ret)
13941                 goto fail;
13942
13943         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
13944         for (i = 0; i < 256; i++) {
13945                 intel_crtc->lut_r[i] = i;
13946                 intel_crtc->lut_g[i] = i;
13947                 intel_crtc->lut_b[i] = i;
13948         }
13949
13950         /*
13951          * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
13952          * is hooked to pipe B. Hence we want plane A feeding pipe B.
13953          */
13954         intel_crtc->pipe = pipe;
13955         intel_crtc->plane = pipe;
13956         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) {
13957                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
13958                 intel_crtc->plane = !pipe;
13959         }
13960
13961         intel_crtc->cursor_base = ~0;
13962         intel_crtc->cursor_cntl = ~0;
13963         intel_crtc->cursor_size = ~0;
13964
13965         intel_crtc->wm.cxsr_allowed = true;
13966
13967         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
13968                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
13969         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
13970         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
13971
13972         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
13973
13974         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
13975         return;
13976
13977 fail:
13978         if (primary)
13979                 drm_plane_cleanup(primary);
13980         if (cursor)
13981                 drm_plane_cleanup(cursor);
13982         kfree(crtc_state);
13983         kfree(intel_crtc);
13984 }
13985
13986 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
13987 {
13988         struct drm_encoder *encoder = connector->base.encoder;
13989         struct drm_device *dev = connector->base.dev;
13990
13991         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
13992
13993         if (!encoder || WARN_ON(!encoder->crtc))
13994                 return INVALID_PIPE;
13995
13996         return to_intel_crtc(encoder->crtc)->pipe;
13997 }
13998
13999 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
14000                                 struct drm_file *file)
14001 {
14002         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14003         struct drm_crtc *drmmode_crtc;
14004         struct intel_crtc *crtc;
14005
14006         drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id);
14007
14008         if (!drmmode_crtc) {
14009                 DRM_ERROR("no such CRTC id\n");
14010                 return -ENOENT;
14011         }
14012
14013         crtc = to_intel_crtc(drmmode_crtc);
14014         pipe_from_crtc_id->pipe = crtc->pipe;
14015
14016         return 0;
14017 }
14018
14019 static int intel_encoder_clones(struct intel_encoder *encoder)
14020 {
14021         struct drm_device *dev = encoder->base.dev;
14022         struct intel_encoder *source_encoder;
14023         int index_mask = 0;
14024         int entry = 0;
14025
14026         for_each_intel_encoder(dev, source_encoder) {
14027                 if (encoders_cloneable(encoder, source_encoder))
14028                         index_mask |= (1 << entry);
14029
14030                 entry++;
14031         }
14032
14033         return index_mask;
14034 }
14035
14036 static bool has_edp_a(struct drm_device *dev)
14037 {
14038         struct drm_i915_private *dev_priv = dev->dev_private;
14039
14040         if (!IS_MOBILE(dev))
14041                 return false;
14042
14043         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14044                 return false;
14045
14046         if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14047                 return false;
14048
14049         return true;
14050 }
14051
14052 static bool intel_crt_present(struct drm_device *dev)
14053 {
14054         struct drm_i915_private *dev_priv = dev->dev_private;
14055
14056         if (INTEL_INFO(dev)->gen >= 9)
14057                 return false;
14058
14059         if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
14060                 return false;
14061
14062         if (IS_CHERRYVIEW(dev))
14063                 return false;
14064
14065         if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support)
14066                 return false;
14067
14068         return true;
14069 }
14070
14071 static void intel_setup_outputs(struct drm_device *dev)
14072 {
14073         struct drm_i915_private *dev_priv = dev->dev_private;
14074         struct intel_encoder *encoder;
14075         bool dpd_is_edp = false;
14076
14077         intel_lvds_init(dev);
14078
14079         if (intel_crt_present(dev))
14080                 intel_crt_init(dev);
14081
14082         if (IS_BROXTON(dev)) {
14083                 /*
14084                  * FIXME: Broxton doesn't support port detection via the
14085                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14086                  * detect the ports.
14087                  */
14088                 intel_ddi_init(dev, PORT_A);
14089                 intel_ddi_init(dev, PORT_B);
14090                 intel_ddi_init(dev, PORT_C);
14091         } else if (HAS_DDI(dev)) {
14092                 int found;
14093
14094                 /*
14095                  * Haswell uses DDI functions to detect digital outputs.
14096                  * On SKL pre-D0 the strap isn't connected, so we assume
14097                  * it's there.
14098                  */
14099                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14100                 /* WaIgnoreDDIAStrap: skl */
14101                 if (found || IS_SKYLAKE(dev))
14102                         intel_ddi_init(dev, PORT_A);
14103
14104                 /* DDI B, C and D detection is indicated by the SFUSE_STRAP
14105                  * register */
14106                 found = I915_READ(SFUSE_STRAP);
14107
14108                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14109                         intel_ddi_init(dev, PORT_B);
14110                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14111                         intel_ddi_init(dev, PORT_C);
14112                 if (found & SFUSE_STRAP_DDID_DETECTED)
14113                         intel_ddi_init(dev, PORT_D);
14114                 /*
14115                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14116                  */
14117                 if (IS_SKYLAKE(dev) &&
14118                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14119                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14120                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14121                         intel_ddi_init(dev, PORT_E);
14122
14123         } else if (HAS_PCH_SPLIT(dev)) {
14124                 int found;
14125                 dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
14126
14127                 if (has_edp_a(dev))
14128                         intel_dp_init(dev, DP_A, PORT_A);
14129
14130                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14131                         /* PCH SDVOB multiplex with HDMIB */
14132                         found = intel_sdvo_init(dev, PCH_SDVOB, true);
14133                         if (!found)
14134                                 intel_hdmi_init(dev, PCH_HDMIB, PORT_B);
14135                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14136                                 intel_dp_init(dev, PCH_DP_B, PORT_B);
14137                 }
14138
14139                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14140                         intel_hdmi_init(dev, PCH_HDMIC, PORT_C);
14141
14142                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14143                         intel_hdmi_init(dev, PCH_HDMID, PORT_D);
14144
14145                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14146                         intel_dp_init(dev, PCH_DP_C, PORT_C);
14147
14148                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14149                         intel_dp_init(dev, PCH_DP_D, PORT_D);
14150         } else if (IS_VALLEYVIEW(dev)) {
14151                 /*
14152                  * The DP_DETECTED bit is the latched state of the DDC
14153                  * SDA pin at boot. However since eDP doesn't require DDC
14154                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14155                  * eDP ports may have been muxed to an alternate function.
14156                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14157                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14158                  * detect eDP ports.
14159                  */
14160                 if (I915_READ(VLV_HDMIB) & SDVO_DETECTED &&
14161                     !intel_dp_is_edp(dev, PORT_B))
14162                         intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
14163                 if (I915_READ(VLV_DP_B) & DP_DETECTED ||
14164                     intel_dp_is_edp(dev, PORT_B))
14165                         intel_dp_init(dev, VLV_DP_B, PORT_B);
14166
14167                 if (I915_READ(VLV_HDMIC) & SDVO_DETECTED &&
14168                     !intel_dp_is_edp(dev, PORT_C))
14169                         intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
14170                 if (I915_READ(VLV_DP_C) & DP_DETECTED ||
14171                     intel_dp_is_edp(dev, PORT_C))
14172                         intel_dp_init(dev, VLV_DP_C, PORT_C);
14173
14174                 if (IS_CHERRYVIEW(dev)) {
14175                         /* eDP not supported on port D, so don't check VBT */
14176                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED)
14177                                 intel_hdmi_init(dev, CHV_HDMID, PORT_D);
14178                         if (I915_READ(CHV_DP_D) & DP_DETECTED)
14179                                 intel_dp_init(dev, CHV_DP_D, PORT_D);
14180                 }
14181
14182                 intel_dsi_init(dev);
14183         } else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) {
14184                 bool found = false;
14185
14186                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14187                         DRM_DEBUG_KMS("probing SDVOB\n");
14188                         found = intel_sdvo_init(dev, GEN3_SDVOB, true);
14189                         if (!found && IS_G4X(dev)) {
14190                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14191                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
14192                         }
14193
14194                         if (!found && IS_G4X(dev))
14195                                 intel_dp_init(dev, DP_B, PORT_B);
14196                 }
14197
14198                 /* Before G4X SDVOC doesn't have its own detect register */
14199
14200                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14201                         DRM_DEBUG_KMS("probing SDVOC\n");
14202                         found = intel_sdvo_init(dev, GEN3_SDVOC, false);
14203                 }
14204
14205                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14206
14207                         if (IS_G4X(dev)) {
14208                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14209                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
14210                         }
14211                         if (IS_G4X(dev))
14212                                 intel_dp_init(dev, DP_C, PORT_C);
14213                 }
14214
14215                 if (IS_G4X(dev) &&
14216                     (I915_READ(DP_D) & DP_DETECTED))
14217                         intel_dp_init(dev, DP_D, PORT_D);
14218         } else if (IS_GEN2(dev))
14219                 intel_dvo_init(dev);
14220
14221         if (SUPPORTS_TV(dev))
14222                 intel_tv_init(dev);
14223
14224         intel_psr_init(dev);
14225
14226         for_each_intel_encoder(dev, encoder) {
14227                 encoder->base.possible_crtcs = encoder->crtc_mask;
14228                 encoder->base.possible_clones =
14229                         intel_encoder_clones(encoder);
14230         }
14231
14232         intel_init_pch_refclk(dev);
14233
14234         drm_helper_move_panel_connectors_to_head(dev);
14235 }
14236
14237 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14238 {
14239         struct drm_device *dev = fb->dev;
14240         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14241
14242         drm_framebuffer_cleanup(fb);
14243         mutex_lock(&dev->struct_mutex);
14244         WARN_ON(!intel_fb->obj->framebuffer_references--);
14245         drm_gem_object_unreference(&intel_fb->obj->base);
14246         mutex_unlock(&dev->struct_mutex);
14247         kfree(intel_fb);
14248 }
14249
14250 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14251                                                 struct drm_file *file,
14252                                                 unsigned int *handle)
14253 {
14254         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14255         struct drm_i915_gem_object *obj = intel_fb->obj;
14256
14257         if (obj->userptr.mm) {
14258                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14259                 return -EINVAL;
14260         }
14261
14262         return drm_gem_handle_create(file, &obj->base, handle);
14263 }
14264
14265 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14266                                         struct drm_file *file,
14267                                         unsigned flags, unsigned color,
14268                                         struct drm_clip_rect *clips,
14269                                         unsigned num_clips)
14270 {
14271         struct drm_device *dev = fb->dev;
14272         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14273         struct drm_i915_gem_object *obj = intel_fb->obj;
14274
14275         mutex_lock(&dev->struct_mutex);
14276         intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
14277         mutex_unlock(&dev->struct_mutex);
14278
14279         return 0;
14280 }
14281
14282 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14283         .destroy = intel_user_framebuffer_destroy,
14284         .create_handle = intel_user_framebuffer_create_handle,
14285         .dirty = intel_user_framebuffer_dirty,
14286 };
14287
14288 static
14289 u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
14290                          uint32_t pixel_format)
14291 {
14292         u32 gen = INTEL_INFO(dev)->gen;
14293
14294         if (gen >= 9) {
14295                 /* "The stride in bytes must not exceed the of the size of 8K
14296                  *  pixels and 32K bytes."
14297                  */
14298                  return min(8192*drm_format_plane_cpp(pixel_format, 0), 32768);
14299         } else if (gen >= 5 && !IS_VALLEYVIEW(dev)) {
14300                 return 32*1024;
14301         } else if (gen >= 4) {
14302                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14303                         return 16*1024;
14304                 else
14305                         return 32*1024;
14306         } else if (gen >= 3) {
14307                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14308                         return 8*1024;
14309                 else
14310                         return 16*1024;
14311         } else {
14312                 /* XXX DSPC is limited to 4k tiled */
14313                 return 8*1024;
14314         }
14315 }
14316
14317 static int intel_framebuffer_init(struct drm_device *dev,
14318                                   struct intel_framebuffer *intel_fb,
14319                                   struct drm_mode_fb_cmd2 *mode_cmd,
14320                                   struct drm_i915_gem_object *obj)
14321 {
14322         unsigned int aligned_height;
14323         int ret;
14324         u32 pitch_limit, stride_alignment;
14325
14326         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
14327
14328         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14329                 /* Enforce that fb modifier and tiling mode match, but only for
14330                  * X-tiled. This is needed for FBC. */
14331                 if (!!(obj->tiling_mode == I915_TILING_X) !=
14332                     !!(mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED)) {
14333                         DRM_DEBUG("tiling_mode doesn't match fb modifier\n");
14334                         return -EINVAL;
14335                 }
14336         } else {
14337                 if (obj->tiling_mode == I915_TILING_X)
14338                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14339                 else if (obj->tiling_mode == I915_TILING_Y) {
14340                         DRM_DEBUG("No Y tiling for legacy addfb\n");
14341                         return -EINVAL;
14342                 }
14343         }
14344
14345         /* Passed in modifier sanity checking. */
14346         switch (mode_cmd->modifier[0]) {
14347         case I915_FORMAT_MOD_Y_TILED:
14348         case I915_FORMAT_MOD_Yf_TILED:
14349                 if (INTEL_INFO(dev)->gen < 9) {
14350                         DRM_DEBUG("Unsupported tiling 0x%llx!\n",
14351                                   mode_cmd->modifier[0]);
14352                         return -EINVAL;
14353                 }
14354         case DRM_FORMAT_MOD_NONE:
14355         case I915_FORMAT_MOD_X_TILED:
14356                 break;
14357         default:
14358                 DRM_DEBUG("Unsupported fb modifier 0x%llx!\n",
14359                           mode_cmd->modifier[0]);
14360                 return -EINVAL;
14361         }
14362
14363         stride_alignment = intel_fb_stride_alignment(dev, mode_cmd->modifier[0],
14364                                                      mode_cmd->pixel_format);
14365         if (mode_cmd->pitches[0] & (stride_alignment - 1)) {
14366                 DRM_DEBUG("pitch (%d) must be at least %u byte aligned\n",
14367                           mode_cmd->pitches[0], stride_alignment);
14368                 return -EINVAL;
14369         }
14370
14371         pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
14372                                            mode_cmd->pixel_format);
14373         if (mode_cmd->pitches[0] > pitch_limit) {
14374                 DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
14375                           mode_cmd->modifier[0] != DRM_FORMAT_MOD_NONE ?
14376                           "tiled" : "linear",
14377                           mode_cmd->pitches[0], pitch_limit);
14378                 return -EINVAL;
14379         }
14380
14381         if (mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED &&
14382             mode_cmd->pitches[0] != obj->stride) {
14383                 DRM_DEBUG("pitch (%d) must match tiling stride (%d)\n",
14384                           mode_cmd->pitches[0], obj->stride);
14385                 return -EINVAL;
14386         }
14387
14388         /* Reject formats not supported by any plane early. */
14389         switch (mode_cmd->pixel_format) {
14390         case DRM_FORMAT_C8:
14391         case DRM_FORMAT_RGB565:
14392         case DRM_FORMAT_XRGB8888:
14393         case DRM_FORMAT_ARGB8888:
14394                 break;
14395         case DRM_FORMAT_XRGB1555:
14396                 if (INTEL_INFO(dev)->gen > 3) {
14397                         DRM_DEBUG("unsupported pixel format: %s\n",
14398                                   drm_get_format_name(mode_cmd->pixel_format));
14399                         return -EINVAL;
14400                 }
14401                 break;
14402         case DRM_FORMAT_ABGR8888:
14403                 if (!IS_VALLEYVIEW(dev) && INTEL_INFO(dev)->gen < 9) {
14404                         DRM_DEBUG("unsupported pixel format: %s\n",
14405                                   drm_get_format_name(mode_cmd->pixel_format));
14406                         return -EINVAL;
14407                 }
14408                 break;
14409         case DRM_FORMAT_XBGR8888:
14410         case DRM_FORMAT_XRGB2101010:
14411         case DRM_FORMAT_XBGR2101010:
14412                 if (INTEL_INFO(dev)->gen < 4) {
14413                         DRM_DEBUG("unsupported pixel format: %s\n",
14414                                   drm_get_format_name(mode_cmd->pixel_format));
14415                         return -EINVAL;
14416                 }
14417                 break;
14418         case DRM_FORMAT_ABGR2101010:
14419                 if (!IS_VALLEYVIEW(dev)) {
14420                         DRM_DEBUG("unsupported pixel format: %s\n",
14421                                   drm_get_format_name(mode_cmd->pixel_format));
14422                         return -EINVAL;
14423                 }
14424                 break;
14425         case DRM_FORMAT_YUYV:
14426         case DRM_FORMAT_UYVY:
14427         case DRM_FORMAT_YVYU:
14428         case DRM_FORMAT_VYUY:
14429                 if (INTEL_INFO(dev)->gen < 5) {
14430                         DRM_DEBUG("unsupported pixel format: %s\n",
14431                                   drm_get_format_name(mode_cmd->pixel_format));
14432                         return -EINVAL;
14433                 }
14434                 break;
14435         default:
14436                 DRM_DEBUG("unsupported pixel format: %s\n",
14437                           drm_get_format_name(mode_cmd->pixel_format));
14438                 return -EINVAL;
14439         }
14440
14441         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14442         if (mode_cmd->offsets[0] != 0)
14443                 return -EINVAL;
14444
14445         aligned_height = intel_fb_align_height(dev, mode_cmd->height,
14446                                                mode_cmd->pixel_format,
14447                                                mode_cmd->modifier[0]);
14448         /* FIXME drm helper for size checks (especially planar formats)? */
14449         if (obj->base.size < aligned_height * mode_cmd->pitches[0])
14450                 return -EINVAL;
14451
14452         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
14453         intel_fb->obj = obj;
14454         intel_fb->obj->framebuffer_references++;
14455
14456         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
14457         if (ret) {
14458                 DRM_ERROR("framebuffer init failed %d\n", ret);
14459                 return ret;
14460         }
14461
14462         return 0;
14463 }
14464
14465 static struct drm_framebuffer *
14466 intel_user_framebuffer_create(struct drm_device *dev,
14467                               struct drm_file *filp,
14468                               struct drm_mode_fb_cmd2 *user_mode_cmd)
14469 {
14470         struct drm_i915_gem_object *obj;
14471         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14472
14473         obj = to_intel_bo(drm_gem_object_lookup(dev, filp,
14474                                                 mode_cmd.handles[0]));
14475         if (&obj->base == NULL)
14476                 return ERR_PTR(-ENOENT);
14477
14478         return intel_framebuffer_create(dev, &mode_cmd, obj);
14479 }
14480
14481 #ifndef CONFIG_DRM_FBDEV_EMULATION
14482 static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
14483 {
14484 }
14485 #endif
14486
14487 static const struct drm_mode_config_funcs intel_mode_funcs = {
14488         .fb_create = intel_user_framebuffer_create,
14489         .output_poll_changed = intel_fbdev_output_poll_changed,
14490         .atomic_check = intel_atomic_check,
14491         .atomic_commit = intel_atomic_commit,
14492         .atomic_state_alloc = intel_atomic_state_alloc,
14493         .atomic_state_clear = intel_atomic_state_clear,
14494 };
14495
14496 /* Set up chip specific display functions */
14497 static void intel_init_display(struct drm_device *dev)
14498 {
14499         struct drm_i915_private *dev_priv = dev->dev_private;
14500
14501         if (HAS_PCH_SPLIT(dev) || IS_G4X(dev))
14502                 dev_priv->display.find_dpll = g4x_find_best_dpll;
14503         else if (IS_CHERRYVIEW(dev))
14504                 dev_priv->display.find_dpll = chv_find_best_dpll;
14505         else if (IS_VALLEYVIEW(dev))
14506                 dev_priv->display.find_dpll = vlv_find_best_dpll;
14507         else if (IS_PINEVIEW(dev))
14508                 dev_priv->display.find_dpll = pnv_find_best_dpll;
14509         else
14510                 dev_priv->display.find_dpll = i9xx_find_best_dpll;
14511
14512         if (INTEL_INFO(dev)->gen >= 9) {
14513                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14514                 dev_priv->display.get_initial_plane_config =
14515                         skylake_get_initial_plane_config;
14516                 dev_priv->display.crtc_compute_clock =
14517                         haswell_crtc_compute_clock;
14518                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14519                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14520                 dev_priv->display.update_primary_plane =
14521                         skylake_update_primary_plane;
14522         } else if (HAS_DDI(dev)) {
14523                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14524                 dev_priv->display.get_initial_plane_config =
14525                         ironlake_get_initial_plane_config;
14526                 dev_priv->display.crtc_compute_clock =
14527                         haswell_crtc_compute_clock;
14528                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14529                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14530                 dev_priv->display.update_primary_plane =
14531                         ironlake_update_primary_plane;
14532         } else if (HAS_PCH_SPLIT(dev)) {
14533                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14534                 dev_priv->display.get_initial_plane_config =
14535                         ironlake_get_initial_plane_config;
14536                 dev_priv->display.crtc_compute_clock =
14537                         ironlake_crtc_compute_clock;
14538                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14539                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14540                 dev_priv->display.update_primary_plane =
14541                         ironlake_update_primary_plane;
14542         } else if (IS_VALLEYVIEW(dev)) {
14543                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14544                 dev_priv->display.get_initial_plane_config =
14545                         i9xx_get_initial_plane_config;
14546                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14547                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14548                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14549                 dev_priv->display.update_primary_plane =
14550                         i9xx_update_primary_plane;
14551         } else {
14552                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14553                 dev_priv->display.get_initial_plane_config =
14554                         i9xx_get_initial_plane_config;
14555                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14556                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14557                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14558                 dev_priv->display.update_primary_plane =
14559                         i9xx_update_primary_plane;
14560         }
14561
14562         /* Returns the core display clock speed */
14563         if (IS_SKYLAKE(dev))
14564                 dev_priv->display.get_display_clock_speed =
14565                         skylake_get_display_clock_speed;
14566         else if (IS_BROXTON(dev))
14567                 dev_priv->display.get_display_clock_speed =
14568                         broxton_get_display_clock_speed;
14569         else if (IS_BROADWELL(dev))
14570                 dev_priv->display.get_display_clock_speed =
14571                         broadwell_get_display_clock_speed;
14572         else if (IS_HASWELL(dev))
14573                 dev_priv->display.get_display_clock_speed =
14574                         haswell_get_display_clock_speed;
14575         else if (IS_VALLEYVIEW(dev))
14576                 dev_priv->display.get_display_clock_speed =
14577                         valleyview_get_display_clock_speed;
14578         else if (IS_GEN5(dev))
14579                 dev_priv->display.get_display_clock_speed =
14580                         ilk_get_display_clock_speed;
14581         else if (IS_I945G(dev) || IS_BROADWATER(dev) ||
14582                  IS_GEN6(dev) || IS_IVYBRIDGE(dev))
14583                 dev_priv->display.get_display_clock_speed =
14584                         i945_get_display_clock_speed;
14585         else if (IS_GM45(dev))
14586                 dev_priv->display.get_display_clock_speed =
14587                         gm45_get_display_clock_speed;
14588         else if (IS_CRESTLINE(dev))
14589                 dev_priv->display.get_display_clock_speed =
14590                         i965gm_get_display_clock_speed;
14591         else if (IS_PINEVIEW(dev))
14592                 dev_priv->display.get_display_clock_speed =
14593                         pnv_get_display_clock_speed;
14594         else if (IS_G33(dev) || IS_G4X(dev))
14595                 dev_priv->display.get_display_clock_speed =
14596                         g33_get_display_clock_speed;
14597         else if (IS_I915G(dev))
14598                 dev_priv->display.get_display_clock_speed =
14599                         i915_get_display_clock_speed;
14600         else if (IS_I945GM(dev) || IS_845G(dev))
14601                 dev_priv->display.get_display_clock_speed =
14602                         i9xx_misc_get_display_clock_speed;
14603         else if (IS_PINEVIEW(dev))
14604                 dev_priv->display.get_display_clock_speed =
14605                         pnv_get_display_clock_speed;
14606         else if (IS_I915GM(dev))
14607                 dev_priv->display.get_display_clock_speed =
14608                         i915gm_get_display_clock_speed;
14609         else if (IS_I865G(dev))
14610                 dev_priv->display.get_display_clock_speed =
14611                         i865_get_display_clock_speed;
14612         else if (IS_I85X(dev))
14613                 dev_priv->display.get_display_clock_speed =
14614                         i85x_get_display_clock_speed;
14615         else { /* 830 */
14616                 WARN(!IS_I830(dev), "Unknown platform. Assuming 133 MHz CDCLK\n");
14617                 dev_priv->display.get_display_clock_speed =
14618                         i830_get_display_clock_speed;
14619         }
14620
14621         if (IS_GEN5(dev)) {
14622                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14623         } else if (IS_GEN6(dev)) {
14624                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14625         } else if (IS_IVYBRIDGE(dev)) {
14626                 /* FIXME: detect B0+ stepping and use auto training */
14627                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14628         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
14629                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14630                 if (IS_BROADWELL(dev)) {
14631                         dev_priv->display.modeset_commit_cdclk =
14632                                 broadwell_modeset_commit_cdclk;
14633                         dev_priv->display.modeset_calc_cdclk =
14634                                 broadwell_modeset_calc_cdclk;
14635                 }
14636         } else if (IS_VALLEYVIEW(dev)) {
14637                 dev_priv->display.modeset_commit_cdclk =
14638                         valleyview_modeset_commit_cdclk;
14639                 dev_priv->display.modeset_calc_cdclk =
14640                         valleyview_modeset_calc_cdclk;
14641         } else if (IS_BROXTON(dev)) {
14642                 dev_priv->display.modeset_commit_cdclk =
14643                         broxton_modeset_commit_cdclk;
14644                 dev_priv->display.modeset_calc_cdclk =
14645                         broxton_modeset_calc_cdclk;
14646         }
14647
14648         switch (INTEL_INFO(dev)->gen) {
14649         case 2:
14650                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
14651                 break;
14652
14653         case 3:
14654                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
14655                 break;
14656
14657         case 4:
14658         case 5:
14659                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
14660                 break;
14661
14662         case 6:
14663                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
14664                 break;
14665         case 7:
14666         case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
14667                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
14668                 break;
14669         case 9:
14670                 /* Drop through - unsupported since execlist only. */
14671         default:
14672                 /* Default just returns -ENODEV to indicate unsupported */
14673                 dev_priv->display.queue_flip = intel_default_queue_flip;
14674         }
14675
14676         mutex_init(&dev_priv->pps_mutex);
14677 }
14678
14679 /*
14680  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
14681  * resume, or other times.  This quirk makes sure that's the case for
14682  * affected systems.
14683  */
14684 static void quirk_pipea_force(struct drm_device *dev)
14685 {
14686         struct drm_i915_private *dev_priv = dev->dev_private;
14687
14688         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
14689         DRM_INFO("applying pipe a force quirk\n");
14690 }
14691
14692 static void quirk_pipeb_force(struct drm_device *dev)
14693 {
14694         struct drm_i915_private *dev_priv = dev->dev_private;
14695
14696         dev_priv->quirks |= QUIRK_PIPEB_FORCE;
14697         DRM_INFO("applying pipe b force quirk\n");
14698 }
14699
14700 /*
14701  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
14702  */
14703 static void quirk_ssc_force_disable(struct drm_device *dev)
14704 {
14705         struct drm_i915_private *dev_priv = dev->dev_private;
14706         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
14707         DRM_INFO("applying lvds SSC disable quirk\n");
14708 }
14709
14710 /*
14711  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
14712  * brightness value
14713  */
14714 static void quirk_invert_brightness(struct drm_device *dev)
14715 {
14716         struct drm_i915_private *dev_priv = dev->dev_private;
14717         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
14718         DRM_INFO("applying inverted panel brightness quirk\n");
14719 }
14720
14721 /* Some VBT's incorrectly indicate no backlight is present */
14722 static void quirk_backlight_present(struct drm_device *dev)
14723 {
14724         struct drm_i915_private *dev_priv = dev->dev_private;
14725         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
14726         DRM_INFO("applying backlight present quirk\n");
14727 }
14728
14729 struct intel_quirk {
14730         int device;
14731         int subsystem_vendor;
14732         int subsystem_device;
14733         void (*hook)(struct drm_device *dev);
14734 };
14735
14736 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
14737 struct intel_dmi_quirk {
14738         void (*hook)(struct drm_device *dev);
14739         const struct dmi_system_id (*dmi_id_list)[];
14740 };
14741
14742 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
14743 {
14744         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
14745         return 1;
14746 }
14747
14748 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
14749         {
14750                 .dmi_id_list = &(const struct dmi_system_id[]) {
14751                         {
14752                                 .callback = intel_dmi_reverse_brightness,
14753                                 .ident = "NCR Corporation",
14754                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
14755                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
14756                                 },
14757                         },
14758                         { }  /* terminating entry */
14759                 },
14760                 .hook = quirk_invert_brightness,
14761         },
14762 };
14763
14764 static struct intel_quirk intel_quirks[] = {
14765         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
14766         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
14767
14768         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
14769         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
14770
14771         /* 830 needs to leave pipe A & dpll A up */
14772         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
14773
14774         /* 830 needs to leave pipe B & dpll B up */
14775         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
14776
14777         /* Lenovo U160 cannot use SSC on LVDS */
14778         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
14779
14780         /* Sony Vaio Y cannot use SSC on LVDS */
14781         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
14782
14783         /* Acer Aspire 5734Z must invert backlight brightness */
14784         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
14785
14786         /* Acer/eMachines G725 */
14787         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
14788
14789         /* Acer/eMachines e725 */
14790         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
14791
14792         /* Acer/Packard Bell NCL20 */
14793         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
14794
14795         /* Acer Aspire 4736Z */
14796         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
14797
14798         /* Acer Aspire 5336 */
14799         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
14800
14801         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
14802         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
14803
14804         /* Acer C720 Chromebook (Core i3 4005U) */
14805         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
14806
14807         /* Apple Macbook 2,1 (Core 2 T7400) */
14808         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
14809
14810         /* Apple Macbook 4,1 */
14811         { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present },
14812
14813         /* Toshiba CB35 Chromebook (Celeron 2955U) */
14814         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
14815
14816         /* HP Chromebook 14 (Celeron 2955U) */
14817         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
14818
14819         /* Dell Chromebook 11 */
14820         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
14821
14822         /* Dell Chromebook 11 (2015 version) */
14823         { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
14824 };
14825
14826 static void intel_init_quirks(struct drm_device *dev)
14827 {
14828         struct pci_dev *d = dev->pdev;
14829         int i;
14830
14831         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
14832                 struct intel_quirk *q = &intel_quirks[i];
14833
14834                 if (d->device == q->device &&
14835                     (d->subsystem_vendor == q->subsystem_vendor ||
14836                      q->subsystem_vendor == PCI_ANY_ID) &&
14837                     (d->subsystem_device == q->subsystem_device ||
14838                      q->subsystem_device == PCI_ANY_ID))
14839                         q->hook(dev);
14840         }
14841         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
14842                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
14843                         intel_dmi_quirks[i].hook(dev);
14844         }
14845 }
14846
14847 /* Disable the VGA plane that we never use */
14848 static void i915_disable_vga(struct drm_device *dev)
14849 {
14850         struct drm_i915_private *dev_priv = dev->dev_private;
14851         u8 sr1;
14852         u32 vga_reg = i915_vgacntrl_reg(dev);
14853
14854         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
14855         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
14856         outb(SR01, VGA_SR_INDEX);
14857         sr1 = inb(VGA_SR_DATA);
14858         outb(sr1 | 1<<5, VGA_SR_DATA);
14859         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
14860         udelay(300);
14861
14862         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
14863         POSTING_READ(vga_reg);
14864 }
14865
14866 void intel_modeset_init_hw(struct drm_device *dev)
14867 {
14868         intel_update_cdclk(dev);
14869         intel_prepare_ddi(dev);
14870         intel_init_clock_gating(dev);
14871         intel_enable_gt_powersave(dev);
14872 }
14873
14874 void intel_modeset_init(struct drm_device *dev)
14875 {
14876         struct drm_i915_private *dev_priv = dev->dev_private;
14877         int sprite, ret;
14878         enum pipe pipe;
14879         struct intel_crtc *crtc;
14880
14881         drm_mode_config_init(dev);
14882
14883         dev->mode_config.min_width = 0;
14884         dev->mode_config.min_height = 0;
14885
14886         dev->mode_config.preferred_depth = 24;
14887         dev->mode_config.prefer_shadow = 1;
14888
14889         dev->mode_config.allow_fb_modifiers = true;
14890
14891         dev->mode_config.funcs = &intel_mode_funcs;
14892
14893         intel_init_quirks(dev);
14894
14895         intel_init_pm(dev);
14896
14897         if (INTEL_INFO(dev)->num_pipes == 0)
14898                 return;
14899
14900         /*
14901          * There may be no VBT; and if the BIOS enabled SSC we can
14902          * just keep using it to avoid unnecessary flicker.  Whereas if the
14903          * BIOS isn't using it, don't assume it will work even if the VBT
14904          * indicates as much.
14905          */
14906         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
14907                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
14908                                             DREF_SSC1_ENABLE);
14909
14910                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
14911                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
14912                                      bios_lvds_use_ssc ? "en" : "dis",
14913                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
14914                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
14915                 }
14916         }
14917
14918         intel_init_display(dev);
14919         intel_init_audio(dev);
14920
14921         if (IS_GEN2(dev)) {
14922                 dev->mode_config.max_width = 2048;
14923                 dev->mode_config.max_height = 2048;
14924         } else if (IS_GEN3(dev)) {
14925                 dev->mode_config.max_width = 4096;
14926                 dev->mode_config.max_height = 4096;
14927         } else {
14928                 dev->mode_config.max_width = 8192;
14929                 dev->mode_config.max_height = 8192;
14930         }
14931
14932         if (IS_845G(dev) || IS_I865G(dev)) {
14933                 dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
14934                 dev->mode_config.cursor_height = 1023;
14935         } else if (IS_GEN2(dev)) {
14936                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
14937                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
14938         } else {
14939                 dev->mode_config.cursor_width = MAX_CURSOR_WIDTH;
14940                 dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT;
14941         }
14942
14943         dev->mode_config.fb_base = dev_priv->gtt.mappable_base;
14944
14945         DRM_DEBUG_KMS("%d display pipe%s available.\n",
14946                       INTEL_INFO(dev)->num_pipes,
14947                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
14948
14949         for_each_pipe(dev_priv, pipe) {
14950                 intel_crtc_init(dev, pipe);
14951                 for_each_sprite(dev_priv, pipe, sprite) {
14952                         ret = intel_plane_init(dev, pipe, sprite);
14953                         if (ret)
14954                                 DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
14955                                               pipe_name(pipe), sprite_name(pipe, sprite), ret);
14956                 }
14957         }
14958
14959         intel_update_czclk(dev_priv);
14960         intel_update_cdclk(dev);
14961
14962         intel_shared_dpll_init(dev);
14963
14964         /* Just disable it once at startup */
14965         i915_disable_vga(dev);
14966         intel_setup_outputs(dev);
14967
14968         /* Just in case the BIOS is doing something questionable. */
14969         intel_fbc_disable(dev_priv);
14970
14971         drm_modeset_lock_all(dev);
14972         intel_modeset_setup_hw_state(dev);
14973         drm_modeset_unlock_all(dev);
14974
14975         for_each_intel_crtc(dev, crtc) {
14976                 struct intel_initial_plane_config plane_config = {};
14977
14978                 if (!crtc->active)
14979                         continue;
14980
14981                 /*
14982                  * Note that reserving the BIOS fb up front prevents us
14983                  * from stuffing other stolen allocations like the ring
14984                  * on top.  This prevents some ugliness at boot time, and
14985                  * can even allow for smooth boot transitions if the BIOS
14986                  * fb is large enough for the active pipe configuration.
14987                  */
14988                 dev_priv->display.get_initial_plane_config(crtc,
14989                                                            &plane_config);
14990
14991                 /*
14992                  * If the fb is shared between multiple heads, we'll
14993                  * just get the first one.
14994                  */
14995                 intel_find_initial_plane_obj(crtc, &plane_config);
14996         }
14997 }
14998
14999 static void intel_enable_pipe_a(struct drm_device *dev)
15000 {
15001         struct intel_connector *connector;
15002         struct drm_connector *crt = NULL;
15003         struct intel_load_detect_pipe load_detect_temp;
15004         struct drm_modeset_acquire_ctx *ctx = dev->mode_config.acquire_ctx;
15005
15006         /* We can't just switch on the pipe A, we need to set things up with a
15007          * proper mode and output configuration. As a gross hack, enable pipe A
15008          * by enabling the load detect pipe once. */
15009         for_each_intel_connector(dev, connector) {
15010                 if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
15011                         crt = &connector->base;
15012                         break;
15013                 }
15014         }
15015
15016         if (!crt)
15017                 return;
15018
15019         if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, ctx))
15020                 intel_release_load_detect_pipe(crt, &load_detect_temp, ctx);
15021 }
15022
15023 static bool
15024 intel_check_plane_mapping(struct intel_crtc *crtc)
15025 {
15026         struct drm_device *dev = crtc->base.dev;
15027         struct drm_i915_private *dev_priv = dev->dev_private;
15028         u32 val;
15029
15030         if (INTEL_INFO(dev)->num_pipes == 1)
15031                 return true;
15032
15033         val = I915_READ(DSPCNTR(!crtc->plane));
15034
15035         if ((val & DISPLAY_PLANE_ENABLE) &&
15036             (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
15037                 return false;
15038
15039         return true;
15040 }
15041
15042 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15043 {
15044         struct drm_device *dev = crtc->base.dev;
15045         struct intel_encoder *encoder;
15046
15047         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15048                 return true;
15049
15050         return false;
15051 }
15052
15053 static void intel_sanitize_crtc(struct intel_crtc *crtc)
15054 {
15055         struct drm_device *dev = crtc->base.dev;
15056         struct drm_i915_private *dev_priv = dev->dev_private;
15057         u32 reg;
15058
15059         /* Clear any frame start delays used for debugging left by the BIOS */
15060         reg = PIPECONF(crtc->config->cpu_transcoder);
15061         I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15062
15063         /* restore vblank interrupts to correct state */
15064         drm_crtc_vblank_reset(&crtc->base);
15065         if (crtc->active) {
15066                 struct intel_plane *plane;
15067
15068                 drm_crtc_vblank_on(&crtc->base);
15069
15070                 /* Disable everything but the primary plane */
15071                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15072                         if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
15073                                 continue;
15074
15075                         plane->disable_plane(&plane->base, &crtc->base);
15076                 }
15077         }
15078
15079         /* We need to sanitize the plane -> pipe mapping first because this will
15080          * disable the crtc (and hence change the state) if it is wrong. Note
15081          * that gen4+ has a fixed plane -> pipe mapping.  */
15082         if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
15083                 bool plane;
15084
15085                 DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n",
15086                               crtc->base.base.id);
15087
15088                 /* Pipe has the wrong plane attached and the plane is active.
15089                  * Temporarily change the plane mapping and disable everything
15090                  * ...  */
15091                 plane = crtc->plane;
15092                 to_intel_plane_state(crtc->base.primary->state)->visible = true;
15093                 crtc->plane = !plane;
15094                 intel_crtc_disable_noatomic(&crtc->base);
15095                 crtc->plane = plane;
15096         }
15097
15098         if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
15099             crtc->pipe == PIPE_A && !crtc->active) {
15100                 /* BIOS forgot to enable pipe A, this mostly happens after
15101                  * resume. Force-enable the pipe to fix this, the update_dpms
15102                  * call below we restore the pipe to the right state, but leave
15103                  * the required bits on. */
15104                 intel_enable_pipe_a(dev);
15105         }
15106
15107         /* Adjust the state of the output pipe according to whether we
15108          * have active connectors/encoders. */
15109         if (!intel_crtc_has_encoders(crtc))
15110                 intel_crtc_disable_noatomic(&crtc->base);
15111
15112         if (crtc->active != crtc->base.state->active) {
15113                 struct intel_encoder *encoder;
15114
15115                 /* This can happen either due to bugs in the get_hw_state
15116                  * functions or because of calls to intel_crtc_disable_noatomic,
15117                  * or because the pipe is force-enabled due to the
15118                  * pipe A quirk. */
15119                 DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was %s, now %s\n",
15120                               crtc->base.base.id,
15121                               crtc->base.state->enable ? "enabled" : "disabled",
15122                               crtc->active ? "enabled" : "disabled");
15123
15124                 WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, NULL) < 0);
15125                 crtc->base.state->active = crtc->active;
15126                 crtc->base.enabled = crtc->active;
15127
15128                 /* Because we only establish the connector -> encoder ->
15129                  * crtc links if something is active, this means the
15130                  * crtc is now deactivated. Break the links. connector
15131                  * -> encoder links are only establish when things are
15132                  *  actually up, hence no need to break them. */
15133                 WARN_ON(crtc->active);
15134
15135                 for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15136                         encoder->base.crtc = NULL;
15137         }
15138
15139         if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
15140                 /*
15141                  * We start out with underrun reporting disabled to avoid races.
15142                  * For correct bookkeeping mark this on active crtcs.
15143                  *
15144                  * Also on gmch platforms we dont have any hardware bits to
15145                  * disable the underrun reporting. Which means we need to start
15146                  * out with underrun reporting disabled also on inactive pipes,
15147                  * since otherwise we'll complain about the garbage we read when
15148                  * e.g. coming up after runtime pm.
15149                  *
15150                  * No protection against concurrent access is required - at
15151                  * worst a fifo underrun happens which also sets this to false.
15152                  */
15153                 crtc->cpu_fifo_underrun_disabled = true;
15154                 crtc->pch_fifo_underrun_disabled = true;
15155         }
15156 }
15157
15158 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15159 {
15160         struct intel_connector *connector;
15161         struct drm_device *dev = encoder->base.dev;
15162         bool active = false;
15163
15164         /* We need to check both for a crtc link (meaning that the
15165          * encoder is active and trying to read from a pipe) and the
15166          * pipe itself being active. */
15167         bool has_active_crtc = encoder->base.crtc &&
15168                 to_intel_crtc(encoder->base.crtc)->active;
15169
15170         for_each_intel_connector(dev, connector) {
15171                 if (connector->base.encoder != &encoder->base)
15172                         continue;
15173
15174                 active = true;
15175                 break;
15176         }
15177
15178         if (active && !has_active_crtc) {
15179                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15180                               encoder->base.base.id,
15181                               encoder->base.name);
15182
15183                 /* Connector is active, but has no active pipe. This is
15184                  * fallout from our resume register restoring. Disable
15185                  * the encoder manually again. */
15186                 if (encoder->base.crtc) {
15187                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15188                                       encoder->base.base.id,
15189                                       encoder->base.name);
15190                         encoder->disable(encoder);
15191                         if (encoder->post_disable)
15192                                 encoder->post_disable(encoder);
15193                 }
15194                 encoder->base.crtc = NULL;
15195
15196                 /* Inconsistent output/port/pipe state happens presumably due to
15197                  * a bug in one of the get_hw_state functions. Or someplace else
15198                  * in our code, like the register restore mess on resume. Clamp
15199                  * things to off as a safer default. */
15200                 for_each_intel_connector(dev, connector) {
15201                         if (connector->encoder != encoder)
15202                                 continue;
15203                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15204                         connector->base.encoder = NULL;
15205                 }
15206         }
15207         /* Enabled encoders without active connectors will be fixed in
15208          * the crtc fixup. */
15209 }
15210
15211 void i915_redisable_vga_power_on(struct drm_device *dev)
15212 {
15213         struct drm_i915_private *dev_priv = dev->dev_private;
15214         u32 vga_reg = i915_vgacntrl_reg(dev);
15215
15216         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15217                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15218                 i915_disable_vga(dev);
15219         }
15220 }
15221
15222 void i915_redisable_vga(struct drm_device *dev)
15223 {
15224         struct drm_i915_private *dev_priv = dev->dev_private;
15225
15226         /* This function can be called both from intel_modeset_setup_hw_state or
15227          * at a very early point in our resume sequence, where the power well
15228          * structures are not yet restored. Since this function is at a very
15229          * paranoid "someone might have enabled VGA while we were not looking"
15230          * level, just check if the power well is enabled instead of trying to
15231          * follow the "don't touch the power well if we don't need it" policy
15232          * the rest of the driver uses. */
15233         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_VGA))
15234                 return;
15235
15236         i915_redisable_vga_power_on(dev);
15237 }
15238
15239 static bool primary_get_hw_state(struct intel_plane *plane)
15240 {
15241         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15242
15243         return I915_READ(DSPCNTR(plane->plane)) & DISPLAY_PLANE_ENABLE;
15244 }
15245
15246 /* FIXME read out full plane state for all planes */
15247 static void readout_plane_state(struct intel_crtc *crtc)
15248 {
15249         struct drm_plane *primary = crtc->base.primary;
15250         struct intel_plane_state *plane_state =
15251                 to_intel_plane_state(primary->state);
15252
15253         plane_state->visible =
15254                 primary_get_hw_state(to_intel_plane(primary));
15255
15256         if (plane_state->visible)
15257                 crtc->base.state->plane_mask |= 1 << drm_plane_index(primary);
15258 }
15259
15260 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15261 {
15262         struct drm_i915_private *dev_priv = dev->dev_private;
15263         enum pipe pipe;
15264         struct intel_crtc *crtc;
15265         struct intel_encoder *encoder;
15266         struct intel_connector *connector;
15267         int i;
15268
15269         for_each_intel_crtc(dev, crtc) {
15270                 __drm_atomic_helper_crtc_destroy_state(&crtc->base, crtc->base.state);
15271                 memset(crtc->config, 0, sizeof(*crtc->config));
15272                 crtc->config->base.crtc = &crtc->base;
15273
15274                 crtc->active = dev_priv->display.get_pipe_config(crtc,
15275                                                                  crtc->config);
15276
15277                 crtc->base.state->active = crtc->active;
15278                 crtc->base.enabled = crtc->active;
15279
15280                 readout_plane_state(crtc);
15281
15282                 DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
15283                               crtc->base.base.id,
15284                               crtc->active ? "enabled" : "disabled");
15285         }
15286
15287         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15288                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15289
15290                 pll->on = pll->get_hw_state(dev_priv, pll,
15291                                             &pll->config.hw_state);
15292                 pll->active = 0;
15293                 pll->config.crtc_mask = 0;
15294                 for_each_intel_crtc(dev, crtc) {
15295                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll) {
15296                                 pll->active++;
15297                                 pll->config.crtc_mask |= 1 << crtc->pipe;
15298                         }
15299                 }
15300
15301                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15302                               pll->name, pll->config.crtc_mask, pll->on);
15303
15304                 if (pll->config.crtc_mask)
15305                         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
15306         }
15307
15308         for_each_intel_encoder(dev, encoder) {
15309                 pipe = 0;
15310
15311                 if (encoder->get_hw_state(encoder, &pipe)) {
15312                         crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15313                         encoder->base.crtc = &crtc->base;
15314                         encoder->get_config(encoder, crtc->config);
15315                 } else {
15316                         encoder->base.crtc = NULL;
15317                 }
15318
15319                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15320                               encoder->base.base.id,
15321                               encoder->base.name,
15322                               encoder->base.crtc ? "enabled" : "disabled",
15323                               pipe_name(pipe));
15324         }
15325
15326         for_each_intel_connector(dev, connector) {
15327                 if (connector->get_hw_state(connector)) {
15328                         connector->base.dpms = DRM_MODE_DPMS_ON;
15329                         connector->base.encoder = &connector->encoder->base;
15330                 } else {
15331                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15332                         connector->base.encoder = NULL;
15333                 }
15334                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15335                               connector->base.base.id,
15336                               connector->base.name,
15337                               connector->base.encoder ? "enabled" : "disabled");
15338         }
15339
15340         for_each_intel_crtc(dev, crtc) {
15341                 crtc->base.hwmode = crtc->config->base.adjusted_mode;
15342
15343                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15344                 if (crtc->base.state->active) {
15345                         intel_mode_from_pipe_config(&crtc->base.mode, crtc->config);
15346                         intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config);
15347                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15348
15349                         /*
15350                          * The initial mode needs to be set in order to keep
15351                          * the atomic core happy. It wants a valid mode if the
15352                          * crtc's enabled, so we do the above call.
15353                          *
15354                          * At this point some state updated by the connectors
15355                          * in their ->detect() callback has not run yet, so
15356                          * no recalculation can be done yet.
15357                          *
15358                          * Even if we could do a recalculation and modeset
15359                          * right now it would cause a double modeset if
15360                          * fbdev or userspace chooses a different initial mode.
15361                          *
15362                          * If that happens, someone indicated they wanted a
15363                          * mode change, which means it's safe to do a full
15364                          * recalculation.
15365                          */
15366                         crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED;
15367
15368                         drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
15369                         update_scanline_offset(crtc);
15370                 }
15371         }
15372 }
15373
15374 /* Scan out the current hw modeset state,
15375  * and sanitizes it to the current state
15376  */
15377 static void
15378 intel_modeset_setup_hw_state(struct drm_device *dev)
15379 {
15380         struct drm_i915_private *dev_priv = dev->dev_private;
15381         enum pipe pipe;
15382         struct intel_crtc *crtc;
15383         struct intel_encoder *encoder;
15384         int i;
15385
15386         intel_modeset_readout_hw_state(dev);
15387
15388         /* HW state is read out, now we need to sanitize this mess. */
15389         for_each_intel_encoder(dev, encoder) {
15390                 intel_sanitize_encoder(encoder);
15391         }
15392
15393         for_each_pipe(dev_priv, pipe) {
15394                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15395                 intel_sanitize_crtc(crtc);
15396                 intel_dump_pipe_config(crtc, crtc->config,
15397                                        "[setup_hw_state]");
15398         }
15399
15400         intel_modeset_update_connector_atomic_state(dev);
15401
15402         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15403                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15404
15405                 if (!pll->on || pll->active)
15406                         continue;
15407
15408                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", pll->name);
15409
15410                 pll->disable(dev_priv, pll);
15411                 pll->on = false;
15412         }
15413
15414         if (IS_VALLEYVIEW(dev))
15415                 vlv_wm_get_hw_state(dev);
15416         else if (IS_GEN9(dev))
15417                 skl_wm_get_hw_state(dev);
15418         else if (HAS_PCH_SPLIT(dev))
15419                 ilk_wm_get_hw_state(dev);
15420
15421         for_each_intel_crtc(dev, crtc) {
15422                 unsigned long put_domains;
15423
15424                 put_domains = modeset_get_crtc_power_domains(&crtc->base);
15425                 if (WARN_ON(put_domains))
15426                         modeset_put_power_domains(dev_priv, put_domains);
15427         }
15428         intel_display_set_init_power(dev_priv, false);
15429 }
15430
15431 void intel_display_resume(struct drm_device *dev)
15432 {
15433         struct drm_atomic_state *state = drm_atomic_state_alloc(dev);
15434         struct intel_connector *conn;
15435         struct intel_plane *plane;
15436         struct drm_crtc *crtc;
15437         int ret;
15438
15439         if (!state)
15440                 return;
15441
15442         state->acquire_ctx = dev->mode_config.acquire_ctx;
15443
15444         /* preserve complete old state, including dpll */
15445         intel_atomic_get_shared_dpll_state(state);
15446
15447         for_each_crtc(dev, crtc) {
15448                 struct drm_crtc_state *crtc_state =
15449                         drm_atomic_get_crtc_state(state, crtc);
15450
15451                 ret = PTR_ERR_OR_ZERO(crtc_state);
15452                 if (ret)
15453                         goto err;
15454
15455                 /* force a restore */
15456                 crtc_state->mode_changed = true;
15457         }
15458
15459         for_each_intel_plane(dev, plane) {
15460                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_plane_state(state, &plane->base));
15461                 if (ret)
15462                         goto err;
15463         }
15464
15465         for_each_intel_connector(dev, conn) {
15466                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(state, &conn->base));
15467                 if (ret)
15468                         goto err;
15469         }
15470
15471         intel_modeset_setup_hw_state(dev);
15472
15473         i915_redisable_vga(dev);
15474         ret = drm_atomic_commit(state);
15475         if (!ret)
15476                 return;
15477
15478 err:
15479         DRM_ERROR("Restoring old state failed with %i\n", ret);
15480         drm_atomic_state_free(state);
15481 }
15482
15483 void intel_modeset_gem_init(struct drm_device *dev)
15484 {
15485         struct drm_crtc *c;
15486         struct drm_i915_gem_object *obj;
15487         int ret;
15488
15489         mutex_lock(&dev->struct_mutex);
15490         intel_init_gt_powersave(dev);
15491         mutex_unlock(&dev->struct_mutex);
15492
15493         intel_modeset_init_hw(dev);
15494
15495         intel_setup_overlay(dev);
15496
15497         /*
15498          * Make sure any fbs we allocated at startup are properly
15499          * pinned & fenced.  When we do the allocation it's too early
15500          * for this.
15501          */
15502         for_each_crtc(dev, c) {
15503                 obj = intel_fb_obj(c->primary->fb);
15504                 if (obj == NULL)
15505                         continue;
15506
15507                 mutex_lock(&dev->struct_mutex);
15508                 ret = intel_pin_and_fence_fb_obj(c->primary,
15509                                                  c->primary->fb,
15510                                                  c->primary->state,
15511                                                  NULL, NULL);
15512                 mutex_unlock(&dev->struct_mutex);
15513                 if (ret) {
15514                         DRM_ERROR("failed to pin boot fb on pipe %d\n",
15515                                   to_intel_crtc(c)->pipe);
15516                         drm_framebuffer_unreference(c->primary->fb);
15517                         c->primary->fb = NULL;
15518                         c->primary->crtc = c->primary->state->crtc = NULL;
15519                         update_state_fb(c->primary);
15520                         c->state->plane_mask &= ~(1 << drm_plane_index(c->primary));
15521                 }
15522         }
15523
15524         intel_backlight_register(dev);
15525 }
15526
15527 void intel_connector_unregister(struct intel_connector *intel_connector)
15528 {
15529         struct drm_connector *connector = &intel_connector->base;
15530
15531         intel_panel_destroy_backlight(connector);
15532         drm_connector_unregister(connector);
15533 }
15534
15535 void intel_modeset_cleanup(struct drm_device *dev)
15536 {
15537         struct drm_i915_private *dev_priv = dev->dev_private;
15538         struct drm_connector *connector;
15539
15540         intel_disable_gt_powersave(dev);
15541
15542         intel_backlight_unregister(dev);
15543
15544         /*
15545          * Interrupts and polling as the first thing to avoid creating havoc.
15546          * Too much stuff here (turning of connectors, ...) would
15547          * experience fancy races otherwise.
15548          */
15549         intel_irq_uninstall(dev_priv);
15550
15551         /*
15552          * Due to the hpd irq storm handling the hotplug work can re-arm the
15553          * poll handlers. Hence disable polling after hpd handling is shut down.
15554          */
15555         drm_kms_helper_poll_fini(dev);
15556
15557         intel_unregister_dsm_handler();
15558
15559         intel_fbc_disable(dev_priv);
15560
15561         /* flush any delayed tasks or pending work */
15562         flush_scheduled_work();
15563
15564         /* destroy the backlight and sysfs files before encoders/connectors */
15565         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
15566                 struct intel_connector *intel_connector;
15567
15568                 intel_connector = to_intel_connector(connector);
15569                 intel_connector->unregister(intel_connector);
15570         }
15571
15572         drm_mode_config_cleanup(dev);
15573
15574         intel_cleanup_overlay(dev);
15575
15576         mutex_lock(&dev->struct_mutex);
15577         intel_cleanup_gt_powersave(dev);
15578         mutex_unlock(&dev->struct_mutex);
15579
15580         intel_teardown_gmbus(dev);
15581 }
15582
15583 /*
15584  * Return which encoder is currently attached for connector.
15585  */
15586 struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
15587 {
15588         return &intel_attached_encoder(connector)->base;
15589 }
15590
15591 void intel_connector_attach_encoder(struct intel_connector *connector,
15592                                     struct intel_encoder *encoder)
15593 {
15594         connector->encoder = encoder;
15595         drm_mode_connector_attach_encoder(&connector->base,
15596                                           &encoder->base);
15597 }
15598
15599 /*
15600  * set vga decode state - true == enable VGA decode
15601  */
15602 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
15603 {
15604         struct drm_i915_private *dev_priv = dev->dev_private;
15605         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
15606         u16 gmch_ctrl;
15607
15608         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
15609                 DRM_ERROR("failed to read control word\n");
15610                 return -EIO;
15611         }
15612
15613         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
15614                 return 0;
15615
15616         if (state)
15617                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
15618         else
15619                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
15620
15621         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
15622                 DRM_ERROR("failed to write control word\n");
15623                 return -EIO;
15624         }
15625
15626         return 0;
15627 }
15628
15629 struct intel_display_error_state {
15630
15631         u32 power_well_driver;
15632
15633         int num_transcoders;
15634
15635         struct intel_cursor_error_state {
15636                 u32 control;
15637                 u32 position;
15638                 u32 base;
15639                 u32 size;
15640         } cursor[I915_MAX_PIPES];
15641
15642         struct intel_pipe_error_state {
15643                 bool power_domain_on;
15644                 u32 source;
15645                 u32 stat;
15646         } pipe[I915_MAX_PIPES];
15647
15648         struct intel_plane_error_state {
15649                 u32 control;
15650                 u32 stride;
15651                 u32 size;
15652                 u32 pos;
15653                 u32 addr;
15654                 u32 surface;
15655                 u32 tile_offset;
15656         } plane[I915_MAX_PIPES];
15657
15658         struct intel_transcoder_error_state {
15659                 bool power_domain_on;
15660                 enum transcoder cpu_transcoder;
15661
15662                 u32 conf;
15663
15664                 u32 htotal;
15665                 u32 hblank;
15666                 u32 hsync;
15667                 u32 vtotal;
15668                 u32 vblank;
15669                 u32 vsync;
15670         } transcoder[4];
15671 };
15672
15673 struct intel_display_error_state *
15674 intel_display_capture_error_state(struct drm_device *dev)
15675 {
15676         struct drm_i915_private *dev_priv = dev->dev_private;
15677         struct intel_display_error_state *error;
15678         int transcoders[] = {
15679                 TRANSCODER_A,
15680                 TRANSCODER_B,
15681                 TRANSCODER_C,
15682                 TRANSCODER_EDP,
15683         };
15684         int i;
15685
15686         if (INTEL_INFO(dev)->num_pipes == 0)
15687                 return NULL;
15688
15689         error = kzalloc(sizeof(*error), GFP_ATOMIC);
15690         if (error == NULL)
15691                 return NULL;
15692
15693         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15694                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
15695
15696         for_each_pipe(dev_priv, i) {
15697                 error->pipe[i].power_domain_on =
15698                         __intel_display_power_is_enabled(dev_priv,
15699                                                          POWER_DOMAIN_PIPE(i));
15700                 if (!error->pipe[i].power_domain_on)
15701                         continue;
15702
15703                 error->cursor[i].control = I915_READ(CURCNTR(i));
15704                 error->cursor[i].position = I915_READ(CURPOS(i));
15705                 error->cursor[i].base = I915_READ(CURBASE(i));
15706
15707                 error->plane[i].control = I915_READ(DSPCNTR(i));
15708                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
15709                 if (INTEL_INFO(dev)->gen <= 3) {
15710                         error->plane[i].size = I915_READ(DSPSIZE(i));
15711                         error->plane[i].pos = I915_READ(DSPPOS(i));
15712                 }
15713                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15714                         error->plane[i].addr = I915_READ(DSPADDR(i));
15715                 if (INTEL_INFO(dev)->gen >= 4) {
15716                         error->plane[i].surface = I915_READ(DSPSURF(i));
15717                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
15718                 }
15719
15720                 error->pipe[i].source = I915_READ(PIPESRC(i));
15721
15722                 if (HAS_GMCH_DISPLAY(dev))
15723                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
15724         }
15725
15726         error->num_transcoders = INTEL_INFO(dev)->num_pipes;
15727         if (HAS_DDI(dev_priv->dev))
15728                 error->num_transcoders++; /* Account for eDP. */
15729
15730         for (i = 0; i < error->num_transcoders; i++) {
15731                 enum transcoder cpu_transcoder = transcoders[i];
15732
15733                 error->transcoder[i].power_domain_on =
15734                         __intel_display_power_is_enabled(dev_priv,
15735                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
15736                 if (!error->transcoder[i].power_domain_on)
15737                         continue;
15738
15739                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
15740
15741                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
15742                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
15743                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
15744                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
15745                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
15746                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
15747                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
15748         }
15749
15750         return error;
15751 }
15752
15753 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
15754
15755 void
15756 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
15757                                 struct drm_device *dev,
15758                                 struct intel_display_error_state *error)
15759 {
15760         struct drm_i915_private *dev_priv = dev->dev_private;
15761         int i;
15762
15763         if (!error)
15764                 return;
15765
15766         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
15767         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15768                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
15769                            error->power_well_driver);
15770         for_each_pipe(dev_priv, i) {
15771                 err_printf(m, "Pipe [%d]:\n", i);
15772                 err_printf(m, "  Power: %s\n",
15773                            error->pipe[i].power_domain_on ? "on" : "off");
15774                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
15775                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
15776
15777                 err_printf(m, "Plane [%d]:\n", i);
15778                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
15779                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
15780                 if (INTEL_INFO(dev)->gen <= 3) {
15781                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
15782                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
15783                 }
15784                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15785                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
15786                 if (INTEL_INFO(dev)->gen >= 4) {
15787                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
15788                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
15789                 }
15790
15791                 err_printf(m, "Cursor [%d]:\n", i);
15792                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
15793                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
15794                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
15795         }
15796
15797         for (i = 0; i < error->num_transcoders; i++) {
15798                 err_printf(m, "CPU transcoder: %c\n",
15799                            transcoder_name(error->transcoder[i].cpu_transcoder));
15800                 err_printf(m, "  Power: %s\n",
15801                            error->transcoder[i].power_domain_on ? "on" : "off");
15802                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
15803                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
15804                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
15805                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
15806                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
15807                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
15808                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
15809         }
15810 }
15811
15812 void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file)
15813 {
15814         struct intel_crtc *crtc;
15815
15816         for_each_intel_crtc(dev, crtc) {
15817                 struct intel_unpin_work *work;
15818
15819                 spin_lock_irq(&dev->event_lock);
15820
15821                 work = crtc->unpin_work;
15822
15823                 if (work && work->event &&
15824                     work->event->base.file_priv == file) {
15825                         kfree(work->event);
15826                         work->event = NULL;
15827                 }
15828
15829                 spin_unlock_irq(&dev->event_lock);
15830         }
15831 }