kernel/drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  42 {
  43         return active == !policy_is_inactive(policy);
  44 }
  45
  46 /* Finds Next Acive/Inactive policy */
  47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  48                                           bool active)
  49 {
  50         do {
  51                 policy = list_next_entry(policy, policy_list);
  52
  53                 /* No more policies in the list */
  54                 if (&policy->policy_list == &cpufreq_policy_list)
  55                         return NULL;
  56         } while (!suitable_policy(policy, active));
  57
  58         return policy;
  59 }
  60
  61 static struct cpufreq_policy *first_policy(bool active)
  62 {
  63         struct cpufreq_policy *policy;
  64
  65         /* No policies in the list */
  66         if (list_empty(&cpufreq_policy_list))
  67                 return NULL;
  68
  69         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  70                                   policy_list);
  71
  72         if (!suitable_policy(policy, active))
  73                 policy = next_policy(policy, active);
  74
  75         return policy;
  76 }
  77
  78 /* Macros to iterate over CPU policies */
  79 #define for_each_suitable_policy(__policy, __active)    \
  80         for (__policy = first_policy(__active);         \
  81              __policy;                                  \
  82              __policy = next_policy(__policy, __active))
  83
  84 #define for_each_active_policy(__policy)                \
  85         for_each_suitable_policy(__policy, true)
  86 #define for_each_inactive_policy(__policy)              \
  87         for_each_suitable_policy(__policy, false)
  88
  89 #define for_each_policy(__policy)                       \
  90         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  91
  92 /* Iterate over governors */
  93 static LIST_HEAD(cpufreq_governor_list);
  94 #define for_each_governor(__governor)                           \
  95         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  96
  97 /**
  98  * The "cpufreq driver" - the arch- or hardware-dependent low
  99  * level driver of CPUFreq support, and its spinlock. This lock
 100  * also protects the cpufreq_cpu_data array.
 101  */
 102 static struct cpufreq_driver *cpufreq_driver;
 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 104 static DEFINE_RWLOCK(cpufreq_driver_lock);
 105 DEFINE_MUTEX(cpufreq_governor_lock);
 106
 107 /* Flag to suspend/resume CPUFreq governors */
 108 static bool cpufreq_suspended;
 109
 110 static inline bool has_target(void)
 111 {
 112         return cpufreq_driver->target_index || cpufreq_driver->target;
 113 }
 114
 115 /* internal prototypes */
 116 static int __cpufreq_governor(struct cpufreq_policy *policy,
 117                 unsigned int event);
 118 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 119 static void handle_update(struct work_struct *work);
 120
 121 /**
 122  * Two notifier lists: the "policy" list is involved in the
 123  * validation process for a new CPU frequency policy; the
 124  * "transition" list for kernel code that needs to handle
 125  * changes to devices when the CPU clock speed changes.
 126  * The mutex locks both lists.
 127  */
 128 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 129 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 130
 131 static bool init_cpufreq_transition_notifier_list_called;
 132 static int __init init_cpufreq_transition_notifier_list(void)
 133 {
 134         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 135         init_cpufreq_transition_notifier_list_called = true;
 136         return 0;
 137 }
 138 pure_initcall(init_cpufreq_transition_notifier_list);
 139
 140 static int off __read_mostly;
 141 static int cpufreq_disabled(void)
 142 {
 143         return off;
 144 }
 145 void disable_cpufreq(void)
 146 {
 147         off = 1;
 148 }
 149 static DEFINE_MUTEX(cpufreq_governor_mutex);
 150
 151 bool have_governor_per_policy(void)
 152 {
 153         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 154 }
 155 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 156
 157 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 158 {
 159         if (have_governor_per_policy())
 160                 return &policy->kobj;
 161         else
 162                 return cpufreq_global_kobject;
 163 }
 164 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 165
 166 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
 167 {
 168         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 169
 170         return policy && !policy_is_inactive(policy) ?
 171                 policy->freq_table : NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
 174
 175 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 176 {
 177         u64 idle_time;
 178         u64 cur_wall_time;
 179         u64 busy_time;
 180
 181         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 182
 183         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 184         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 185         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 186         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 187         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 188         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 189
 190         idle_time = cur_wall_time - busy_time;
 191         if (wall)
 192                 *wall = cputime_to_usecs(cur_wall_time);
 193
 194         return cputime_to_usecs(idle_time);
 195 }
 196
 197 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 198 {
 199         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 200
 201         if (idle_time == -1ULL)
 202                 return get_cpu_idle_time_jiffy(cpu, wall);
 203         else if (!io_busy)
 204                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 205
 206         return idle_time;
 207 }
 208 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 209
 210 /*
 211  * This is a generic cpufreq init() routine which can be used by cpufreq
 212  * drivers of SMP systems. It will do following:
 213  * - validate & show freq table passed
 214  * - set policies transition latency
 215  * - policy->cpus with all possible CPUs
 216  */
 217 int cpufreq_generic_init(struct cpufreq_policy *policy,
 218                 struct cpufreq_frequency_table *table,
 219                 unsigned int transition_latency)
 220 {
 221         int ret;
 222
 223         ret = cpufreq_table_validate_and_show(policy, table);
 224         if (ret) {
 225                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 226                 return ret;
 227         }
 228
 229         policy->cpuinfo.transition_latency = transition_latency;
 230
 231         /*
 232          * The driver only supports the SMP configuration where all processors
 233          * share the clock and voltage and clock.
 234          */
 235         cpumask_setall(policy->cpus);
 236
 237         return 0;
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 240
 241 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 242 {
 243         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 244
 245         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 246 }
 247 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 248
 249 unsigned int cpufreq_generic_get(unsigned int cpu)
 250 {
 251         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 252
 253         if (!policy || IS_ERR(policy->clk)) {
 254                 pr_err("%s: No %s associated to cpu: %d\n",
 255                        __func__, policy ? "clk" : "policy", cpu);
 256                 return 0;
 257         }
 258
 259         return clk_get_rate(policy->clk) / 1000;
 260 }
 261 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 262
 263 /**
 264  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 265  *
 266  * @cpu: cpu to find policy for.
 267  *
 268  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 269  * It also increments the kobject reference count to mark it busy and so would
 270  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 271  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 272  * freed as that depends on the kobj count.
 273  *
 274  * Return: A valid policy on success, otherwise NULL on failure.
 275  */
 276 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 277 {
 278         struct cpufreq_policy *policy = NULL;
 279         unsigned long flags;
 280
 281         if (WARN_ON(cpu >= nr_cpu_ids))
 282                 return NULL;
 283
 284         /* get the cpufreq driver */
 285         read_lock_irqsave(&cpufreq_driver_lock, flags);
 286
 287         if (cpufreq_driver) {
 288                 /* get the CPU */
 289                 policy = cpufreq_cpu_get_raw(cpu);
 290                 if (policy)
 291                         kobject_get(&policy->kobj);
 292         }
 293
 294         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 295
 296         return policy;
 297 }
 298 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 299
 300 /**
 301  * cpufreq_cpu_put: Decrements the usage count of a policy
 302  *
 303  * @policy: policy earlier returned by cpufreq_cpu_get().
 304  *
 305  * This decrements the kobject reference count incremented earlier by calling
 306  * cpufreq_cpu_get().
 307  */
 308 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 309 {
 310         kobject_put(&policy->kobj);
 311 }
 312 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 313
 314 /*********************************************************************
 315  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 316  *********************************************************************/
 317
 318 /**
 319  * adjust_jiffies - adjust the system "loops_per_jiffy"
 320  *
 321  * This function alters the system "loops_per_jiffy" for the clock
 322  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 323  * systems as each CPU might be scaled differently. So, use the arch
 324  * per-CPU loops_per_jiffy value wherever possible.
 325  */
 326 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 327 {
 328 #ifndef CONFIG_SMP
 329         static unsigned long l_p_j_ref;
 330         static unsigned int l_p_j_ref_freq;
 331
 332         if (ci->flags & CPUFREQ_CONST_LOOPS)
 333                 return;
 334
 335         if (!l_p_j_ref_freq) {
 336                 l_p_j_ref = loops_per_jiffy;
 337                 l_p_j_ref_freq = ci->old;
 338                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 339                          l_p_j_ref, l_p_j_ref_freq);
 340         }
 341         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 342                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 343                                                                 ci->new);
 344                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 345                          loops_per_jiffy, ci->new);
 346         }
 347 #endif
 348 }
 349
 350 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 351                 struct cpufreq_freqs *freqs, unsigned int state)
 352 {
 353         BUG_ON(irqs_disabled());
 354
 355         if (cpufreq_disabled())
 356                 return;
 357
 358         freqs->flags = cpufreq_driver->flags;
 359         pr_debug("notification %u of frequency transition to %u kHz\n",
 360                  state, freqs->new);
 361
 362         switch (state) {
 363
 364         case CPUFREQ_PRECHANGE:
 365                 /* detect if the driver reported a value as "old frequency"
 366                  * which is not equal to what the cpufreq core thinks is
 367                  * "old frequency".
 368                  */
 369                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 370                         if ((policy) && (policy->cpu == freqs->cpu) &&
 371                             (policy->cur) && (policy->cur != freqs->old)) {
 372                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 373                                          freqs->old, policy->cur);
 374                                 freqs->old = policy->cur;
 375                         }
 376                 }
 377                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 378                                 CPUFREQ_PRECHANGE, freqs);
 379                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 380                 break;
 381
 382         case CPUFREQ_POSTCHANGE:
 383                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 384                 pr_debug("FREQ: %lu - CPU: %lu\n",
 385                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 386                 trace_cpu_frequency(freqs->new, freqs->cpu);
 387                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 388                                 CPUFREQ_POSTCHANGE, freqs);
 389                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 390                         policy->cur = freqs->new;
 391                 break;
 392         }
 393 }
 394
 395 /**
 396  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 397  * on frequency transition.
 398  *
 399  * This function calls the transition notifiers and the "adjust_jiffies"
 400  * function. It is called twice on all CPU frequency changes that have
 401  * external effects.
 402  */
 403 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, unsigned int state)
 405 {
 406         for_each_cpu(freqs->cpu, policy->cpus)
 407                 __cpufreq_notify_transition(policy, freqs, state);
 408 }
 409
 410 /* Do post notifications when there are chances that transition has failed */
 411 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 412                 struct cpufreq_freqs *freqs, int transition_failed)
 413 {
 414         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 415         if (!transition_failed)
 416                 return;
 417
 418         swap(freqs->old, freqs->new);
 419         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 420         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 421 }
 422
 423 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 424                 struct cpufreq_freqs *freqs)
 425 {
 426
 427         /*
 428          * Catch double invocations of _begin() which lead to self-deadlock.
 429          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 430          * doesn't invoke _begin() on their behalf, and hence the chances of
 431          * double invocations are very low. Moreover, there are scenarios
 432          * where these checks can emit false-positive warnings in these
 433          * drivers; so we avoid that by skipping them altogether.
 434          */
 435         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 436                                 && current == policy->transition_task);
 437
 438 wait:
 439         wait_event(policy->transition_wait, !policy->transition_ongoing);
 440
 441         spin_lock(&policy->transition_lock);
 442
 443         if (unlikely(policy->transition_ongoing)) {
 444                 spin_unlock(&policy->transition_lock);
 445                 goto wait;
 446         }
 447
 448         policy->transition_ongoing = true;
 449         policy->transition_task = current;
 450
 451         spin_unlock(&policy->transition_lock);
 452
 453         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 454 }
 455 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 456
 457 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 458                 struct cpufreq_freqs *freqs, int transition_failed)
 459 {
 460         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 461                 return;
 462
 463         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 464
 465         policy->transition_ongoing = false;
 466         policy->transition_task = NULL;
 467
 468         wake_up(&policy->transition_wait);
 469 }
 470 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 471
 472
 473 /*********************************************************************
 474  *                          SYSFS INTERFACE                          *
 475  *********************************************************************/
 476 static ssize_t show_boost(struct kobject *kobj,
 477                                  struct attribute *attr, char *buf)
 478 {
 479         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 480 }
 481
 482 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 483                                   const char *buf, size_t count)
 484 {
 485         int ret, enable;
 486
 487         ret = sscanf(buf, "%d", &enable);
 488         if (ret != 1 || enable < 0 || enable > 1)
 489                 return -EINVAL;
 490
 491         if (cpufreq_boost_trigger_state(enable)) {
 492                 pr_err("%s: Cannot %s BOOST!\n",
 493                        __func__, enable ? "enable" : "disable");
 494                 return -EINVAL;
 495         }
 496
 497         pr_debug("%s: cpufreq BOOST %s\n",
 498                  __func__, enable ? "enabled" : "disabled");
 499
 500         return count;
 501 }
 502 define_one_global_rw(boost);
 503
 504 static struct cpufreq_governor *find_governor(const char *str_governor)
 505 {
 506         struct cpufreq_governor *t;
 507
 508         for_each_governor(t)
 509                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 510                         return t;
 511
 512         return NULL;
 513 }
 514
 515 /**
 516  * cpufreq_parse_governor - parse a governor string
 517  */
 518 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 519                                 struct cpufreq_governor **governor)
 520 {
 521         int err = -EINVAL;
 522
 523         if (cpufreq_driver->setpolicy) {
 524                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 525                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 526                         err = 0;
 527                 } else if (!strncasecmp(str_governor, "powersave",
 528                                                 CPUFREQ_NAME_LEN)) {
 529                         *policy = CPUFREQ_POLICY_POWERSAVE;
 530                         err = 0;
 531                 }
 532         } else {
 533                 struct cpufreq_governor *t;
 534
 535                 mutex_lock(&cpufreq_governor_mutex);
 536
 537                 t = find_governor(str_governor);
 538
 539                 if (t == NULL) {
 540                         int ret;
 541
 542                         mutex_unlock(&cpufreq_governor_mutex);
 543                         ret = request_module("cpufreq_%s", str_governor);
 544                         mutex_lock(&cpufreq_governor_mutex);
 545
 546                         if (ret == 0)
 547                                 t = find_governor(str_governor);
 548                 }
 549
 550                 if (t != NULL) {
 551                         *governor = t;
 552                         err = 0;
 553                 }
 554
 555                 mutex_unlock(&cpufreq_governor_mutex);
 556         }
 557         return err;
 558 }
 559
 560 /**
 561  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 562  * print out cpufreq information
 563  *
 564  * Write out information from cpufreq_driver->policy[cpu]; object must be
 565  * "unsigned int".
 566  */
 567
 568 #define show_one(file_name, object)                     \
 569 static ssize_t show_##file_name                         \
 570 (struct cpufreq_policy *policy, char *buf)              \
 571 {                                                       \
 572         return sprintf(buf, "%u\n", policy->object);    \
 573 }
 574
 575 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 576 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 577 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 578 show_one(scaling_min_freq, min);
 579 show_one(scaling_max_freq, max);
 580
 581 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 582 {
 583         ssize_t ret;
 584
 585         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 586                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 587         else
 588                 ret = sprintf(buf, "%u\n", policy->cur);
 589         return ret;
 590 }
 591
 592 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 593                                 struct cpufreq_policy *new_policy);
 594
 595 /**
 596  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 597  */
 598 #define store_one(file_name, object)                    \
 599 static ssize_t store_##file_name                                        \
 600 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 601 {                                                                       \
 602         int ret, temp;                                                  \
 603         struct cpufreq_policy new_policy;                               \
 604                                                                         \
 605         memcpy(&new_policy, policy, sizeof(*policy));                   \
 606                                                                         \
 607         ret = sscanf(buf, "%u", &new_policy.object);                    \
 608         if (ret != 1)                                                   \
 609                 return -EINVAL;                                         \
 610                                                                         \
 611         temp = new_policy.object;                                       \
 612         ret = cpufreq_set_policy(policy, &new_policy);          \
 613         if (!ret)                                                       \
 614                 policy->user_policy.object = temp;                      \
 615                                                                         \
 616         return ret ? ret : count;                                       \
 617 }
 618
 619 store_one(scaling_min_freq, min);
 620 store_one(scaling_max_freq, max);
 621
 622 /**
 623  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 624  */
 625 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 626                                         char *buf)
 627 {
 628         unsigned int cur_freq = __cpufreq_get(policy);
 629         if (!cur_freq)
 630                 return sprintf(buf, "<unknown>");
 631         return sprintf(buf, "%u\n", cur_freq);
 632 }
 633
 634 /**
 635  * show_scaling_governor - show the current policy for the specified CPU
 636  */
 637 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 638 {
 639         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 640                 return sprintf(buf, "powersave\n");
 641         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 642                 return sprintf(buf, "performance\n");
 643         else if (policy->governor)
 644                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 645                                 policy->governor->name);
 646         return -EINVAL;
 647 }
 648
 649 /**
 650  * store_scaling_governor - store policy for the specified CPU
 651  */
 652 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 653                                         const char *buf, size_t count)
 654 {
 655         int ret;
 656         char    str_governor[16];
 657         struct cpufreq_policy new_policy;
 658
 659         memcpy(&new_policy, policy, sizeof(*policy));
 660
 661         ret = sscanf(buf, "%15s", str_governor);
 662         if (ret != 1)
 663                 return -EINVAL;
 664
 665         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 666                                                 &new_policy.governor))
 667                 return -EINVAL;
 668
 669         ret = cpufreq_set_policy(policy, &new_policy);
 670         return ret ? ret : count;
 671 }
 672
 673 /**
 674  * show_scaling_driver - show the cpufreq driver currently loaded
 675  */
 676 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 677 {
 678         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 679 }
 680
 681 /**
 682  * show_scaling_available_governors - show the available CPUfreq governors
 683  */
 684 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 685                                                 char *buf)
 686 {
 687         ssize_t i = 0;
 688         struct cpufreq_governor *t;
 689
 690         if (!has_target()) {
 691                 i += sprintf(buf, "performance powersave");
 692                 goto out;
 693         }
 694
 695         for_each_governor(t) {
 696                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 697                     - (CPUFREQ_NAME_LEN + 2)))
 698                         goto out;
 699                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 700         }
 701 out:
 702         i += sprintf(&buf[i], "\n");
 703         return i;
 704 }
 705
 706 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 707 {
 708         ssize_t i = 0;
 709         unsigned int cpu;
 710
 711         for_each_cpu(cpu, mask) {
 712                 if (i)
 713                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 714                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 715                 if (i >= (PAGE_SIZE - 5))
 716                         break;
 717         }
 718         i += sprintf(&buf[i], "\n");
 719         return i;
 720 }
 721 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 722
 723 /**
 724  * show_related_cpus - show the CPUs affected by each transition even if
 725  * hw coordination is in use
 726  */
 727 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 728 {
 729         return cpufreq_show_cpus(policy->related_cpus, buf);
 730 }
 731
 732 /**
 733  * show_affected_cpus - show the CPUs affected by each transition
 734  */
 735 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 736 {
 737         return cpufreq_show_cpus(policy->cpus, buf);
 738 }
 739
 740 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 741                                         const char *buf, size_t count)
 742 {
 743         unsigned int freq = 0;
 744         unsigned int ret;
 745
 746         if (!policy->governor || !policy->governor->store_setspeed)
 747                 return -EINVAL;
 748
 749         ret = sscanf(buf, "%u", &freq);
 750         if (ret != 1)
 751                 return -EINVAL;
 752
 753         policy->governor->store_setspeed(policy, freq);
 754
 755         return count;
 756 }
 757
 758 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 759 {
 760         if (!policy->governor || !policy->governor->show_setspeed)
 761                 return sprintf(buf, "<unsupported>\n");
 762
 763         return policy->governor->show_setspeed(policy, buf);
 764 }
 765
 766 /**
 767  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 768  */
 769 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 770 {
 771         unsigned int limit;
 772         int ret;
 773         if (cpufreq_driver->bios_limit) {
 774                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 775                 if (!ret)
 776                         return sprintf(buf, "%u\n", limit);
 777         }
 778         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 779 }
 780
 781 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 782 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 783 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 784 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 785 cpufreq_freq_attr_ro(scaling_available_governors);
 786 cpufreq_freq_attr_ro(scaling_driver);
 787 cpufreq_freq_attr_ro(scaling_cur_freq);
 788 cpufreq_freq_attr_ro(bios_limit);
 789 cpufreq_freq_attr_ro(related_cpus);
 790 cpufreq_freq_attr_ro(affected_cpus);
 791 cpufreq_freq_attr_rw(scaling_min_freq);
 792 cpufreq_freq_attr_rw(scaling_max_freq);
 793 cpufreq_freq_attr_rw(scaling_governor);
 794 cpufreq_freq_attr_rw(scaling_setspeed);
 795
 796 static struct attribute *default_attrs[] = {
 797         &cpuinfo_min_freq.attr,
 798         &cpuinfo_max_freq.attr,
 799         &cpuinfo_transition_latency.attr,
 800         &scaling_min_freq.attr,
 801         &scaling_max_freq.attr,
 802         &affected_cpus.attr,
 803         &related_cpus.attr,
 804         &scaling_governor.attr,
 805         &scaling_driver.attr,
 806         &scaling_available_governors.attr,
 807         &scaling_setspeed.attr,
 808         NULL
 809 };
 810
 811 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 812 #define to_attr(a) container_of(a, struct freq_attr, attr)
 813
 814 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 815 {
 816         struct cpufreq_policy *policy = to_policy(kobj);
 817         struct freq_attr *fattr = to_attr(attr);
 818         ssize_t ret;
 819
 820         down_read(&policy->rwsem);
 821
 822         if (fattr->show)
 823                 ret = fattr->show(policy, buf);
 824         else
 825                 ret = -EIO;
 826
 827         up_read(&policy->rwsem);
 828
 829         return ret;
 830 }
 831
 832 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 833                      const char *buf, size_t count)
 834 {
 835         struct cpufreq_policy *policy = to_policy(kobj);
 836         struct freq_attr *fattr = to_attr(attr);
 837         ssize_t ret = -EINVAL;
 838
 839         get_online_cpus();
 840
 841         if (!cpu_online(policy->cpu))
 842                 goto unlock;
 843
 844         down_write(&policy->rwsem);
 845
 846         if (fattr->store)
 847                 ret = fattr->store(policy, buf, count);
 848         else
 849                 ret = -EIO;
 850
 851         up_write(&policy->rwsem);
 852 unlock:
 853         put_online_cpus();
 854
 855         return ret;
 856 }
 857
 858 static void cpufreq_sysfs_release(struct kobject *kobj)
 859 {
 860         struct cpufreq_policy *policy = to_policy(kobj);
 861         pr_debug("last reference is dropped\n");
 862         complete(&policy->kobj_unregister);
 863 }
 864
 865 static const struct sysfs_ops sysfs_ops = {
 866         .show   = show,
 867         .store  = store,
 868 };
 869
 870 static struct kobj_type ktype_cpufreq = {
 871         .sysfs_ops      = &sysfs_ops,
 872         .default_attrs  = default_attrs,
 873         .release        = cpufreq_sysfs_release,
 874 };
 875
 876 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 877 {
 878         struct device *cpu_dev;
 879
 880         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 881
 882         if (!policy)
 883                 return 0;
 884
 885         cpu_dev = get_cpu_device(cpu);
 886         if (WARN_ON(!cpu_dev))
 887                 return 0;
 888
 889         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 890 }
 891
 892 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 893 {
 894         struct device *cpu_dev;
 895
 896         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 897
 898         cpu_dev = get_cpu_device(cpu);
 899         if (WARN_ON(!cpu_dev))
 900                 return;
 901
 902         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 903 }
 904
 905 /* Add/remove symlinks for all related CPUs */
 906 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 907 {
 908         unsigned int j;
 909         int ret = 0;
 910
 911         /* Some related CPUs might not be present (physically hotplugged) */
 912         for_each_cpu(j, policy->real_cpus) {
 913                 ret = add_cpu_dev_symlink(policy, j);
 914                 if (ret)
 915                         break;
 916         }
 917
 918         return ret;
 919 }
 920
 921 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
 922 {
 923         unsigned int j;
 924
 925         /* Some related CPUs might not be present (physically hotplugged) */
 926         for_each_cpu(j, policy->real_cpus)
 927                 remove_cpu_dev_symlink(policy, j);
 928 }
 929
 930 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 931 {
 932         struct freq_attr **drv_attr;
 933         int ret = 0;
 934
 935         /* set up files for this cpu device */
 936         drv_attr = cpufreq_driver->attr;
 937         while (drv_attr && *drv_attr) {
 938                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 939                 if (ret)
 940                         return ret;
 941                 drv_attr++;
 942         }
 943         if (cpufreq_driver->get) {
 944                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 945                 if (ret)
 946                         return ret;
 947         }
 948
 949         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 950         if (ret)
 951                 return ret;
 952
 953         if (cpufreq_driver->bios_limit) {
 954                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 955                 if (ret)
 956                         return ret;
 957         }
 958
 959         return cpufreq_add_dev_symlink(policy);
 960 }
 961
 962 static int cpufreq_init_policy(struct cpufreq_policy *policy)
 963 {
 964         struct cpufreq_governor *gov = NULL;
 965         struct cpufreq_policy new_policy;
 966
 967         memcpy(&new_policy, policy, sizeof(*policy));
 968
 969         /* Update governor of new_policy to the governor used before hotplug */
 970         gov = find_governor(policy->last_governor);
 971         if (gov)
 972                 pr_debug("Restoring governor %s for cpu %d\n",
 973                                 policy->governor->name, policy->cpu);
 974         else
 975                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 976
 977         new_policy.governor = gov;
 978
 979         /* Use the default policy if there is no last_policy. */
 980         if (cpufreq_driver->setpolicy) {
 981                 if (policy->last_policy)
 982                         new_policy.policy = policy->last_policy;
 983                 else
 984                         cpufreq_parse_governor(gov->name, &new_policy.policy,
 985                                                NULL);
 986         }
 987         /* set default policy */
 988         return cpufreq_set_policy(policy, &new_policy);
 989 }
 990
 991 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 992 {
 993         int ret = 0;
 994
 995         /* Has this CPU been taken care of already? */
 996         if (cpumask_test_cpu(cpu, policy->cpus))
 997                 return 0;
 998
 999         if (has_target()) {
1000                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1001                 if (ret) {
1002                         pr_err("%s: Failed to stop governor\n", __func__);
1003                         return ret;
1004                 }
1005         }
1006
1007         down_write(&policy->rwsem);
1008         cpumask_set_cpu(cpu, policy->cpus);
1009         up_write(&policy->rwsem);
1010
1011         if (has_target()) {
1012                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1013                 if (!ret)
1014                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1015
1016                 if (ret) {
1017                         pr_err("%s: Failed to start governor\n", __func__);
1018                         return ret;
1019                 }
1020         }
1021
1022         return 0;
1023 }
1024
1025 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1026 {
1027         struct device *dev = get_cpu_device(cpu);
1028         struct cpufreq_policy *policy;
1029
1030         if (WARN_ON(!dev))
1031                 return NULL;
1032
1033         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1034         if (!policy)
1035                 return NULL;
1036
1037         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1038                 goto err_free_policy;
1039
1040         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1041                 goto err_free_cpumask;
1042
1043         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1044                 goto err_free_rcpumask;
1045
1046         kobject_init(&policy->kobj, &ktype_cpufreq);
1047         INIT_LIST_HEAD(&policy->policy_list);
1048         init_rwsem(&policy->rwsem);
1049         spin_lock_init(&policy->transition_lock);
1050         init_waitqueue_head(&policy->transition_wait);
1051         init_completion(&policy->kobj_unregister);
1052         INIT_WORK(&policy->update, handle_update);
1053
1054         policy->cpu = cpu;
1055         return policy;
1056
1057 err_free_rcpumask:
1058         free_cpumask_var(policy->related_cpus);
1059 err_free_cpumask:
1060         free_cpumask_var(policy->cpus);
1061 err_free_policy:
1062         kfree(policy);
1063
1064         return NULL;
1065 }
1066
1067 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1068 {
1069         struct kobject *kobj;
1070         struct completion *cmp;
1071
1072         if (notify)
1073                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1074                                              CPUFREQ_REMOVE_POLICY, policy);
1075
1076         down_write(&policy->rwsem);
1077         cpufreq_remove_dev_symlink(policy);
1078         kobj = &policy->kobj;
1079         cmp = &policy->kobj_unregister;
1080         up_write(&policy->rwsem);
1081         kobject_put(kobj);
1082
1083         /*
1084          * We need to make sure that the underlying kobj is
1085          * actually not referenced anymore by anybody before we
1086          * proceed with unloading.
1087          */
1088         pr_debug("waiting for dropping of refcount\n");
1089         wait_for_completion(cmp);
1090         pr_debug("wait complete\n");
1091 }
1092
1093 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1094 {
1095         unsigned long flags;
1096         int cpu;
1097
1098         /* Remove policy from list */
1099         write_lock_irqsave(&cpufreq_driver_lock, flags);
1100         list_del(&policy->policy_list);
1101
1102         for_each_cpu(cpu, policy->related_cpus)
1103                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1104         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1105
1106         cpufreq_policy_put_kobj(policy, notify);
1107         free_cpumask_var(policy->real_cpus);
1108         free_cpumask_var(policy->related_cpus);
1109         free_cpumask_var(policy->cpus);
1110         kfree(policy);
1111 }
1112
1113 static int cpufreq_online(unsigned int cpu)
1114 {
1115         struct cpufreq_policy *policy;
1116         bool new_policy;
1117         unsigned long flags;
1118         unsigned int j;
1119         int ret;
1120
1121         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1122
1123         /* Check if this CPU already has a policy to manage it */
1124         policy = per_cpu(cpufreq_cpu_data, cpu);
1125         if (policy) {
1126                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1127                 if (!policy_is_inactive(policy))
1128                         return cpufreq_add_policy_cpu(policy, cpu);
1129
1130                 /* This is the only online CPU for the policy.  Start over. */
1131                 new_policy = false;
1132                 down_write(&policy->rwsem);
1133                 policy->cpu = cpu;
1134                 policy->governor = NULL;
1135                 up_write(&policy->rwsem);
1136         } else {
1137                 new_policy = true;
1138                 policy = cpufreq_policy_alloc(cpu);
1139                 if (!policy)
1140                         return -ENOMEM;
1141         }
1142
1143         cpumask_copy(policy->cpus, cpumask_of(cpu));
1144
1145         /* call driver. From then on the cpufreq must be able
1146          * to accept all calls to ->verify and ->setpolicy for this CPU
1147          */
1148         ret = cpufreq_driver->init(policy);
1149         if (ret) {
1150                 pr_debug("initialization failed\n");
1151                 goto out_free_policy;
1152         }
1153
1154         down_write(&policy->rwsem);
1155
1156         if (new_policy) {
1157                 /* related_cpus should at least include policy->cpus. */
1158                 cpumask_copy(policy->related_cpus, policy->cpus);
1159                 /* Remember CPUs present at the policy creation time. */
1160                 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1161
1162                 /* Name and add the kobject */
1163                 ret = kobject_add(&policy->kobj, cpufreq_global_kobject,
1164                                   "policy%u",
1165                                   cpumask_first(policy->related_cpus));
1166                 if (ret) {
1167                         pr_err("%s: failed to add policy->kobj: %d\n", __func__,
1168                                ret);
1169                         goto out_exit_policy;
1170                 }
1171         }
1172
1173         /*
1174          * affected cpus must always be the one, which are online. We aren't
1175          * managing offline cpus here.
1176          */
1177         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1178
1179         if (new_policy) {
1180                 policy->user_policy.min = policy->min;
1181                 policy->user_policy.max = policy->max;
1182
1183                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1184                 for_each_cpu(j, policy->related_cpus)
1185                         per_cpu(cpufreq_cpu_data, j) = policy;
1186                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1187         }
1188
1189         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1190                 policy->cur = cpufreq_driver->get(policy->cpu);
1191                 if (!policy->cur) {
1192                         pr_err("%s: ->get() failed\n", __func__);
1193                         goto out_exit_policy;
1194                 }
1195         }
1196
1197         /*
1198          * Sometimes boot loaders set CPU frequency to a value outside of
1199          * frequency table present with cpufreq core. In such cases CPU might be
1200          * unstable if it has to run on that frequency for long duration of time
1201          * and so its better to set it to a frequency which is specified in
1202          * freq-table. This also makes cpufreq stats inconsistent as
1203          * cpufreq-stats would fail to register because current frequency of CPU
1204          * isn't found in freq-table.
1205          *
1206          * Because we don't want this change to effect boot process badly, we go
1207          * for the next freq which is >= policy->cur ('cur' must be set by now,
1208          * otherwise we will end up setting freq to lowest of the table as 'cur'
1209          * is initialized to zero).
1210          *
1211          * We are passing target-freq as "policy->cur - 1" otherwise
1212          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1213          * equal to target-freq.
1214          */
1215         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1216             && has_target()) {
1217                 /* Are we running at unknown frequency ? */
1218                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1219                 if (ret == -EINVAL) {
1220                         /* Warn user and fix it */
1221                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1222                                 __func__, policy->cpu, policy->cur);
1223                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1224                                 CPUFREQ_RELATION_L);
1225
1226                         /*
1227                          * Reaching here after boot in a few seconds may not
1228                          * mean that system will remain stable at "unknown"
1229                          * frequency for longer duration. Hence, a BUG_ON().
1230                          */
1231                         BUG_ON(ret);
1232                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1233                                 __func__, policy->cpu, policy->cur);
1234                 }
1235         }
1236
1237         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1238                                      CPUFREQ_START, policy);
1239
1240         if (new_policy) {
1241                 ret = cpufreq_add_dev_interface(policy);
1242                 if (ret)
1243                         goto out_exit_policy;
1244                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1245                                 CPUFREQ_CREATE_POLICY, policy);
1246
1247                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1248                 list_add(&policy->policy_list, &cpufreq_policy_list);
1249                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1250         }
1251
1252         ret = cpufreq_init_policy(policy);
1253         if (ret) {
1254                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1255                        __func__, cpu, ret);
1256                 /* cpufreq_policy_free() will notify based on this */
1257                 new_policy = false;
1258                 goto out_exit_policy;
1259         }
1260
1261         up_write(&policy->rwsem);
1262
1263         kobject_uevent(&policy->kobj, KOBJ_ADD);
1264
1265         /* Callback for handling stuff after policy is ready */
1266         if (cpufreq_driver->ready)
1267                 cpufreq_driver->ready(policy);
1268
1269         pr_debug("initialization complete\n");
1270
1271         return 0;
1272
1273 out_exit_policy:
1274         up_write(&policy->rwsem);
1275
1276         if (cpufreq_driver->exit)
1277                 cpufreq_driver->exit(policy);
1278 out_free_policy:
1279         cpufreq_policy_free(policy, !new_policy);
1280         return ret;
1281 }
1282
1283 /**
1284  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1285  * @dev: CPU device.
1286  * @sif: Subsystem interface structure pointer (not used)
1287  */
1288 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1289 {
1290         unsigned cpu = dev->id;
1291         int ret;
1292
1293         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1294
1295         if (cpu_online(cpu)) {
1296                 ret = cpufreq_online(cpu);
1297         } else {
1298                 /*
1299                  * A hotplug notifier will follow and we will handle it as CPU
1300                  * online then.  For now, just create the sysfs link, unless
1301                  * there is no policy or the link is already present.
1302                  */
1303                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1304
1305                 ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1306                         ? add_cpu_dev_symlink(policy, cpu) : 0;
1307         }
1308
1309         return ret;
1310 }
1311
1312 static void cpufreq_offline_prepare(unsigned int cpu)
1313 {
1314         struct cpufreq_policy *policy;
1315
1316         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1317
1318         policy = cpufreq_cpu_get_raw(cpu);
1319         if (!policy) {
1320                 pr_debug("%s: No cpu_data found\n", __func__);
1321                 return;
1322         }
1323
1324         if (has_target()) {
1325                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1326                 if (ret)
1327                         pr_err("%s: Failed to stop governor\n", __func__);
1328         }
1329
1330         down_write(&policy->rwsem);
1331         cpumask_clear_cpu(cpu, policy->cpus);
1332
1333         if (policy_is_inactive(policy)) {
1334                 if (has_target())
1335                         strncpy(policy->last_governor, policy->governor->name,
1336                                 CPUFREQ_NAME_LEN);
1337                 else
1338                         policy->last_policy = policy->policy;
1339         } else if (cpu == policy->cpu) {
1340                 /* Nominate new CPU */
1341                 policy->cpu = cpumask_any(policy->cpus);
1342         }
1343         up_write(&policy->rwsem);
1344
1345         /* Start governor again for active policy */
1346         if (!policy_is_inactive(policy)) {
1347                 if (has_target()) {
1348                         int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1349                         if (!ret)
1350                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1351
1352                         if (ret)
1353                                 pr_err("%s: Failed to start governor\n", __func__);
1354                 }
1355         } else if (cpufreq_driver->stop_cpu) {
1356                 cpufreq_driver->stop_cpu(policy);
1357         }
1358 }
1359
1360 static void cpufreq_offline_finish(unsigned int cpu)
1361 {
1362         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1363
1364         if (!policy) {
1365                 pr_debug("%s: No cpu_data found\n", __func__);
1366                 return;
1367         }
1368
1369         /* Only proceed for inactive policies */
1370         if (!policy_is_inactive(policy))
1371                 return;
1372
1373         /* If cpu is last user of policy, free policy */
1374         if (has_target()) {
1375                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1376                 if (ret)
1377                         pr_err("%s: Failed to exit governor\n", __func__);
1378         }
1379
1380         /*
1381          * Perform the ->exit() even during light-weight tear-down,
1382          * since this is a core component, and is essential for the
1383          * subsequent light-weight ->init() to succeed.
1384          */
1385         if (cpufreq_driver->exit) {
1386                 cpufreq_driver->exit(policy);
1387                 policy->freq_table = NULL;
1388         }
1389 }
1390
1391 /**
1392  * cpufreq_remove_dev - remove a CPU device
1393  *
1394  * Removes the cpufreq interface for a CPU device.
1395  */
1396 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1397 {
1398         unsigned int cpu = dev->id;
1399         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1400
1401         if (!policy)
1402                 return;
1403
1404         if (cpu_online(cpu)) {
1405                 cpufreq_offline_prepare(cpu);
1406                 cpufreq_offline_finish(cpu);
1407         }
1408
1409         cpumask_clear_cpu(cpu, policy->real_cpus);
1410         remove_cpu_dev_symlink(policy, cpu);
1411
1412         if (cpumask_empty(policy->real_cpus))
1413                 cpufreq_policy_free(policy, true);
1414 }
1415
1416 static void handle_update(struct work_struct *work)
1417 {
1418         struct cpufreq_policy *policy =
1419                 container_of(work, struct cpufreq_policy, update);
1420         unsigned int cpu = policy->cpu;
1421         pr_debug("handle_update for cpu %u called\n", cpu);
1422         cpufreq_update_policy(cpu);
1423 }
1424
1425 /**
1426  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1427  *      in deep trouble.
1428  *      @policy: policy managing CPUs
1429  *      @new_freq: CPU frequency the CPU actually runs at
1430  *
1431  *      We adjust to current frequency first, and need to clean up later.
1432  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1433  */
1434 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1435                                 unsigned int new_freq)
1436 {
1437         struct cpufreq_freqs freqs;
1438
1439         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1440                  policy->cur, new_freq);
1441
1442         freqs.old = policy->cur;
1443         freqs.new = new_freq;
1444
1445         cpufreq_freq_transition_begin(policy, &freqs);
1446         cpufreq_freq_transition_end(policy, &freqs, 0);
1447 }
1448
1449 /**
1450  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1451  * @cpu: CPU number
1452  *
1453  * This is the last known freq, without actually getting it from the driver.
1454  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1455  */
1456 unsigned int cpufreq_quick_get(unsigned int cpu)
1457 {
1458         struct cpufreq_policy *policy;
1459         unsigned int ret_freq = 0;
1460
1461         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1462                 return cpufreq_driver->get(cpu);
1463
1464         policy = cpufreq_cpu_get(cpu);
1465         if (policy) {
1466                 ret_freq = policy->cur;
1467                 cpufreq_cpu_put(policy);
1468         }
1469
1470         return ret_freq;
1471 }
1472 EXPORT_SYMBOL(cpufreq_quick_get);
1473
1474 /**
1475  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1476  * @cpu: CPU number
1477  *
1478  * Just return the max possible frequency for a given CPU.
1479  */
1480 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1481 {
1482         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1483         unsigned int ret_freq = 0;
1484
1485         if (policy) {
1486                 ret_freq = policy->max;
1487                 cpufreq_cpu_put(policy);
1488         }
1489
1490         return ret_freq;
1491 }
1492 EXPORT_SYMBOL(cpufreq_quick_get_max);
1493
1494 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1495 {
1496         unsigned int ret_freq = 0;
1497
1498         if (!cpufreq_driver->get)
1499                 return ret_freq;
1500
1501         ret_freq = cpufreq_driver->get(policy->cpu);
1502
1503         /* Updating inactive policies is invalid, so avoid doing that. */
1504         if (unlikely(policy_is_inactive(policy)))
1505                 return ret_freq;
1506
1507         if (ret_freq && policy->cur &&
1508                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1509                 /* verify no discrepancy between actual and
1510                                         saved value exists */
1511                 if (unlikely(ret_freq != policy->cur)) {
1512                         cpufreq_out_of_sync(policy, ret_freq);
1513                         schedule_work(&policy->update);
1514                 }
1515         }
1516
1517         return ret_freq;
1518 }
1519
1520 /**
1521  * cpufreq_get - get the current CPU frequency (in kHz)
1522  * @cpu: CPU number
1523  *
1524  * Get the CPU current (static) CPU frequency
1525  */
1526 unsigned int cpufreq_get(unsigned int cpu)
1527 {
1528         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1529         unsigned int ret_freq = 0;
1530
1531         if (policy) {
1532                 down_read(&policy->rwsem);
1533                 ret_freq = __cpufreq_get(policy);
1534                 up_read(&policy->rwsem);
1535
1536                 cpufreq_cpu_put(policy);
1537         }
1538
1539         return ret_freq;
1540 }
1541 EXPORT_SYMBOL(cpufreq_get);
1542
1543 static struct subsys_interface cpufreq_interface = {
1544         .name           = "cpufreq",
1545         .subsys         = &cpu_subsys,
1546         .add_dev        = cpufreq_add_dev,
1547         .remove_dev     = cpufreq_remove_dev,
1548 };
1549
1550 /*
1551  * In case platform wants some specific frequency to be configured
1552  * during suspend..
1553  */
1554 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1555 {
1556         int ret;
1557
1558         if (!policy->suspend_freq) {
1559                 pr_debug("%s: suspend_freq not defined\n", __func__);
1560                 return 0;
1561         }
1562
1563         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1564                         policy->suspend_freq);
1565
1566         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1567                         CPUFREQ_RELATION_H);
1568         if (ret)
1569                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1570                                 __func__, policy->suspend_freq, ret);
1571
1572         return ret;
1573 }
1574 EXPORT_SYMBOL(cpufreq_generic_suspend);
1575
1576 /**
1577  * cpufreq_suspend() - Suspend CPUFreq governors
1578  *
1579  * Called during system wide Suspend/Hibernate cycles for suspending governors
1580  * as some platforms can't change frequency after this point in suspend cycle.
1581  * Because some of the devices (like: i2c, regulators, etc) they use for
1582  * changing frequency are suspended quickly after this point.
1583  */
1584 void cpufreq_suspend(void)
1585 {
1586         struct cpufreq_policy *policy;
1587
1588         if (!cpufreq_driver)
1589                 return;
1590
1591         if (!has_target())
1592                 goto suspend;
1593
1594         pr_debug("%s: Suspending Governors\n", __func__);
1595
1596         for_each_active_policy(policy) {
1597                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1598                         pr_err("%s: Failed to stop governor for policy: %p\n",
1599                                 __func__, policy);
1600                 else if (cpufreq_driver->suspend
1601                     && cpufreq_driver->suspend(policy))
1602                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1603                                 policy);
1604         }
1605
1606 suspend:
1607         cpufreq_suspended = true;
1608 }
1609
1610 /**
1611  * cpufreq_resume() - Resume CPUFreq governors
1612  *
1613  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1614  * are suspended with cpufreq_suspend().
1615  */
1616 void cpufreq_resume(void)
1617 {
1618         struct cpufreq_policy *policy;
1619
1620         if (!cpufreq_driver)
1621                 return;
1622
1623         cpufreq_suspended = false;
1624
1625         if (!has_target())
1626                 return;
1627
1628         pr_debug("%s: Resuming Governors\n", __func__);
1629
1630         for_each_active_policy(policy) {
1631                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1632                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1633                                 policy);
1634                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1635                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1636                         pr_err("%s: Failed to start governor for policy: %p\n",
1637                                 __func__, policy);
1638         }
1639
1640         /*
1641          * schedule call cpufreq_update_policy() for first-online CPU, as that
1642          * wouldn't be hotplugged-out on suspend. It will verify that the
1643          * current freq is in sync with what we believe it to be.
1644          */
1645         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1646         if (WARN_ON(!policy))
1647                 return;
1648
1649         schedule_work(&policy->update);
1650 }
1651
1652 /**
1653  *      cpufreq_get_current_driver - return current driver's name
1654  *
1655  *      Return the name string of the currently loaded cpufreq driver
1656  *      or NULL, if none.
1657  */
1658 const char *cpufreq_get_current_driver(void)
1659 {
1660         if (cpufreq_driver)
1661                 return cpufreq_driver->name;
1662
1663         return NULL;
1664 }
1665 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1666
1667 /**
1668  *      cpufreq_get_driver_data - return current driver data
1669  *
1670  *      Return the private data of the currently loaded cpufreq
1671  *      driver, or NULL if no cpufreq driver is loaded.
1672  */
1673 void *cpufreq_get_driver_data(void)
1674 {
1675         if (cpufreq_driver)
1676                 return cpufreq_driver->driver_data;
1677
1678         return NULL;
1679 }
1680 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1681
1682 /*********************************************************************
1683  *                     NOTIFIER LISTS INTERFACE                      *
1684  *********************************************************************/
1685
1686 /**
1687  *      cpufreq_register_notifier - register a driver with cpufreq
1688  *      @nb: notifier function to register
1689  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1690  *
1691  *      Add a driver to one of two lists: either a list of drivers that
1692  *      are notified about clock rate changes (once before and once after
1693  *      the transition), or a list of drivers that are notified about
1694  *      changes in cpufreq policy.
1695  *
1696  *      This function may sleep, and has the same return conditions as
1697  *      blocking_notifier_chain_register.
1698  */
1699 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1700 {
1701         int ret;
1702
1703         if (cpufreq_disabled())
1704                 return -EINVAL;
1705
1706         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1707
1708         switch (list) {
1709         case CPUFREQ_TRANSITION_NOTIFIER:
1710                 ret = srcu_notifier_chain_register(
1711                                 &cpufreq_transition_notifier_list, nb);
1712                 break;
1713         case CPUFREQ_POLICY_NOTIFIER:
1714                 ret = blocking_notifier_chain_register(
1715                                 &cpufreq_policy_notifier_list, nb);
1716                 break;
1717         default:
1718                 ret = -EINVAL;
1719         }
1720
1721         return ret;
1722 }
1723 EXPORT_SYMBOL(cpufreq_register_notifier);
1724
1725 /**
1726  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1727  *      @nb: notifier block to be unregistered
1728  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1729  *
1730  *      Remove a driver from the CPU frequency notifier list.
1731  *
1732  *      This function may sleep, and has the same return conditions as
1733  *      blocking_notifier_chain_unregister.
1734  */
1735 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1736 {
1737         int ret;
1738
1739         if (cpufreq_disabled())
1740                 return -EINVAL;
1741
1742         switch (list) {
1743         case CPUFREQ_TRANSITION_NOTIFIER:
1744                 ret = srcu_notifier_chain_unregister(
1745                                 &cpufreq_transition_notifier_list, nb);
1746                 break;
1747         case CPUFREQ_POLICY_NOTIFIER:
1748                 ret = blocking_notifier_chain_unregister(
1749                                 &cpufreq_policy_notifier_list, nb);
1750                 break;
1751         default:
1752                 ret = -EINVAL;
1753         }
1754
1755         return ret;
1756 }
1757 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1758
1759
1760 /*********************************************************************
1761  *                              GOVERNORS                            *
1762  *********************************************************************/
1763
1764 /* Must set freqs->new to intermediate frequency */
1765 static int __target_intermediate(struct cpufreq_policy *policy,
1766                                  struct cpufreq_freqs *freqs, int index)
1767 {
1768         int ret;
1769
1770         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1771
1772         /* We don't need to switch to intermediate freq */
1773         if (!freqs->new)
1774                 return 0;
1775
1776         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1777                  __func__, policy->cpu, freqs->old, freqs->new);
1778
1779         cpufreq_freq_transition_begin(policy, freqs);
1780         ret = cpufreq_driver->target_intermediate(policy, index);
1781         cpufreq_freq_transition_end(policy, freqs, ret);
1782
1783         if (ret)
1784                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1785                        __func__, ret);
1786
1787         return ret;
1788 }
1789
1790 static int __target_index(struct cpufreq_policy *policy,
1791                           struct cpufreq_frequency_table *freq_table, int index)
1792 {
1793         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1794         unsigned int intermediate_freq = 0;
1795         int retval = -EINVAL;
1796         bool notify;
1797
1798         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1799         if (notify) {
1800                 /* Handle switching to intermediate frequency */
1801                 if (cpufreq_driver->get_intermediate) {
1802                         retval = __target_intermediate(policy, &freqs, index);
1803                         if (retval)
1804                                 return retval;
1805
1806                         intermediate_freq = freqs.new;
1807                         /* Set old freq to intermediate */
1808                         if (intermediate_freq)
1809                                 freqs.old = freqs.new;
1810                 }
1811
1812                 freqs.new = freq_table[index].frequency;
1813                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1814                          __func__, policy->cpu, freqs.old, freqs.new);
1815
1816                 cpufreq_freq_transition_begin(policy, &freqs);
1817         }
1818
1819         retval = cpufreq_driver->target_index(policy, index);
1820         if (retval)
1821                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1822                        retval);
1823
1824         if (notify) {
1825                 cpufreq_freq_transition_end(policy, &freqs, retval);
1826
1827                 /*
1828                  * Failed after setting to intermediate freq? Driver should have
1829                  * reverted back to initial frequency and so should we. Check
1830                  * here for intermediate_freq instead of get_intermediate, in
1831                  * case we haven't switched to intermediate freq at all.
1832                  */
1833                 if (unlikely(retval && intermediate_freq)) {
1834                         freqs.old = intermediate_freq;
1835                         freqs.new = policy->restore_freq;
1836                         cpufreq_freq_transition_begin(policy, &freqs);
1837                         cpufreq_freq_transition_end(policy, &freqs, 0);
1838                 }
1839         }
1840
1841         return retval;
1842 }
1843
1844 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1845                             unsigned int target_freq,
1846                             unsigned int relation)
1847 {
1848         unsigned int old_target_freq = target_freq;
1849         int retval = -EINVAL;
1850
1851         if (cpufreq_disabled())
1852                 return -ENODEV;
1853
1854         /* Make sure that target_freq is within supported range */
1855         if (target_freq > policy->max)
1856                 target_freq = policy->max;
1857         if (target_freq < policy->min)
1858                 target_freq = policy->min;
1859
1860         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1861                  policy->cpu, target_freq, relation, old_target_freq);
1862
1863         /*
1864          * This might look like a redundant call as we are checking it again
1865          * after finding index. But it is left intentionally for cases where
1866          * exactly same freq is called again and so we can save on few function
1867          * calls.
1868          */
1869         if (target_freq == policy->cur)
1870                 return 0;
1871
1872         /* Save last value to restore later on errors */
1873         policy->restore_freq = policy->cur;
1874
1875         if (cpufreq_driver->target)
1876                 retval = cpufreq_driver->target(policy, target_freq, relation);
1877         else if (cpufreq_driver->target_index) {
1878                 struct cpufreq_frequency_table *freq_table;
1879                 int index;
1880
1881                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1882                 if (unlikely(!freq_table)) {
1883                         pr_err("%s: Unable to find freq_table\n", __func__);
1884                         goto out;
1885                 }
1886
1887                 retval = cpufreq_frequency_table_target(policy, freq_table,
1888                                 target_freq, relation, &index);
1889                 if (unlikely(retval)) {
1890                         pr_err("%s: Unable to find matching freq\n", __func__);
1891                         goto out;
1892                 }
1893
1894                 if (freq_table[index].frequency == policy->cur) {
1895                         retval = 0;
1896                         goto out;
1897                 }
1898
1899                 retval = __target_index(policy, freq_table, index);
1900         }
1901
1902 out:
1903         return retval;
1904 }
1905 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1906
1907 int cpufreq_driver_target(struct cpufreq_policy *policy,
1908                           unsigned int target_freq,
1909                           unsigned int relation)
1910 {
1911         int ret = -EINVAL;
1912
1913         down_write(&policy->rwsem);
1914
1915         ret = __cpufreq_driver_target(policy, target_freq, relation);
1916
1917         up_write(&policy->rwsem);
1918
1919         return ret;
1920 }
1921 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1922
1923 static int __cpufreq_governor(struct cpufreq_policy *policy,
1924                                         unsigned int event)
1925 {
1926         int ret;
1927
1928         /* Only must be defined when default governor is known to have latency
1929            restrictions, like e.g. conservative or ondemand.
1930            That this is the case is already ensured in Kconfig
1931         */
1932 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1933         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1934 #else
1935         struct cpufreq_governor *gov = NULL;
1936 #endif
1937
1938         /* Don't start any governor operations if we are entering suspend */
1939         if (cpufreq_suspended)
1940                 return 0;
1941         /*
1942          * Governor might not be initiated here if ACPI _PPC changed
1943          * notification happened, so check it.
1944          */
1945         if (!policy->governor)
1946                 return -EINVAL;
1947
1948         if (policy->governor->max_transition_latency &&
1949             policy->cpuinfo.transition_latency >
1950             policy->governor->max_transition_latency) {
1951                 if (!gov)
1952                         return -EINVAL;
1953                 else {
1954                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1955                                 policy->governor->name, gov->name);
1956                         policy->governor = gov;
1957                 }
1958         }
1959
1960         if (event == CPUFREQ_GOV_POLICY_INIT)
1961                 if (!try_module_get(policy->governor->owner))
1962                         return -EINVAL;
1963
1964         pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
1965
1966         mutex_lock(&cpufreq_governor_lock);
1967         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1968             || (!policy->governor_enabled
1969             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1970                 mutex_unlock(&cpufreq_governor_lock);
1971                 return -EBUSY;
1972         }
1973
1974         if (event == CPUFREQ_GOV_STOP)
1975                 policy->governor_enabled = false;
1976         else if (event == CPUFREQ_GOV_START)
1977                 policy->governor_enabled = true;
1978
1979         mutex_unlock(&cpufreq_governor_lock);
1980
1981         ret = policy->governor->governor(policy, event);
1982
1983         if (!ret) {
1984                 if (event == CPUFREQ_GOV_POLICY_INIT)
1985                         policy->governor->initialized++;
1986                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1987                         policy->governor->initialized--;
1988         } else {
1989                 /* Restore original values */
1990                 mutex_lock(&cpufreq_governor_lock);
1991                 if (event == CPUFREQ_GOV_STOP)
1992                         policy->governor_enabled = true;
1993                 else if (event == CPUFREQ_GOV_START)
1994                         policy->governor_enabled = false;
1995                 mutex_unlock(&cpufreq_governor_lock);
1996         }
1997
1998         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1999                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2000                 module_put(policy->governor->owner);
2001
2002         return ret;
2003 }
2004
2005 int cpufreq_register_governor(struct cpufreq_governor *governor)
2006 {
2007         int err;
2008
2009         if (!governor)
2010                 return -EINVAL;
2011
2012         if (cpufreq_disabled())
2013                 return -ENODEV;
2014
2015         mutex_lock(&cpufreq_governor_mutex);
2016
2017         governor->initialized = 0;
2018         err = -EBUSY;
2019         if (!find_governor(governor->name)) {
2020                 err = 0;
2021                 list_add(&governor->governor_list, &cpufreq_governor_list);
2022         }
2023
2024         mutex_unlock(&cpufreq_governor_mutex);
2025         return err;
2026 }
2027 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2028
2029 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2030 {
2031         struct cpufreq_policy *policy;
2032         unsigned long flags;
2033
2034         if (!governor)
2035                 return;
2036
2037         if (cpufreq_disabled())
2038                 return;
2039
2040         /* clear last_governor for all inactive policies */
2041         read_lock_irqsave(&cpufreq_driver_lock, flags);
2042         for_each_inactive_policy(policy) {
2043                 if (!strcmp(policy->last_governor, governor->name)) {
2044                         policy->governor = NULL;
2045                         strcpy(policy->last_governor, "\0");
2046                 }
2047         }
2048         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2049
2050         mutex_lock(&cpufreq_governor_mutex);
2051         list_del(&governor->governor_list);
2052         mutex_unlock(&cpufreq_governor_mutex);
2053         return;
2054 }
2055 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2056
2057
2058 /*********************************************************************
2059  *                          POLICY INTERFACE                         *
2060  *********************************************************************/
2061
2062 /**
2063  * cpufreq_get_policy - get the current cpufreq_policy
2064  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2065  *      is written
2066  *
2067  * Reads the current cpufreq policy.
2068  */
2069 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2070 {
2071         struct cpufreq_policy *cpu_policy;
2072         if (!policy)
2073                 return -EINVAL;
2074
2075         cpu_policy = cpufreq_cpu_get(cpu);
2076         if (!cpu_policy)
2077                 return -EINVAL;
2078
2079         memcpy(policy, cpu_policy, sizeof(*policy));
2080
2081         cpufreq_cpu_put(cpu_policy);
2082         return 0;
2083 }
2084 EXPORT_SYMBOL(cpufreq_get_policy);
2085
2086 /*
2087  * policy : current policy.
2088  * new_policy: policy to be set.
2089  */
2090 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2091                                 struct cpufreq_policy *new_policy)
2092 {
2093         struct cpufreq_governor *old_gov;
2094         int ret;
2095
2096         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2097                  new_policy->cpu, new_policy->min, new_policy->max);
2098
2099         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2100
2101         /*
2102         * This check works well when we store new min/max freq attributes,
2103         * because new_policy is a copy of policy with one field updated.
2104         */
2105         if (new_policy->min > new_policy->max)
2106                 return -EINVAL;
2107
2108         /* verify the cpu speed can be set within this limit */
2109         ret = cpufreq_driver->verify(new_policy);
2110         if (ret)
2111                 return ret;
2112
2113         /* adjust if necessary - all reasons */
2114         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2115                         CPUFREQ_ADJUST, new_policy);
2116
2117         /*
2118          * verify the cpu speed can be set within this limit, which might be
2119          * different to the first one
2120          */
2121         ret = cpufreq_driver->verify(new_policy);
2122         if (ret)
2123                 return ret;
2124
2125         /* notification of the new policy */
2126         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2127                         CPUFREQ_NOTIFY, new_policy);
2128
2129         policy->min = new_policy->min;
2130         policy->max = new_policy->max;
2131
2132         pr_debug("new min and max freqs are %u - %u kHz\n",
2133                  policy->min, policy->max);
2134
2135         if (cpufreq_driver->setpolicy) {
2136                 policy->policy = new_policy->policy;
2137                 pr_debug("setting range\n");
2138                 return cpufreq_driver->setpolicy(new_policy);
2139         }
2140
2141         if (new_policy->governor == policy->governor)
2142                 goto out;
2143
2144         pr_debug("governor switch\n");
2145
2146         /* save old, working values */
2147         old_gov = policy->governor;
2148         /* end old governor */
2149         if (old_gov) {
2150                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2151                 if (ret) {
2152                         /* This can happen due to race with other operations */
2153                         pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2154                                  __func__, old_gov->name, ret);
2155                         return ret;
2156                 }
2157
2158                 up_write(&policy->rwsem);
2159                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2160                 down_write(&policy->rwsem);
2161
2162                 if (ret) {
2163                         pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2164                                __func__, old_gov->name, ret);
2165                         return ret;
2166                 }
2167         }
2168
2169         /* start new governor */
2170         policy->governor = new_policy->governor;
2171         ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2172         if (!ret) {
2173                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2174                 if (!ret)
2175                         goto out;
2176
2177                 up_write(&policy->rwsem);
2178                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2179                 down_write(&policy->rwsem);
2180         }
2181
2182         /* new governor failed, so re-start old one */
2183         pr_debug("starting governor %s failed\n", policy->governor->name);
2184         if (old_gov) {
2185                 policy->governor = old_gov;
2186                 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2187                         policy->governor = NULL;
2188                 else
2189                         __cpufreq_governor(policy, CPUFREQ_GOV_START);
2190         }
2191
2192         return ret;
2193
2194  out:
2195         pr_debug("governor: change or update limits\n");
2196         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2197 }
2198
2199 /**
2200  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2201  *      @cpu: CPU which shall be re-evaluated
2202  *
2203  *      Useful for policy notifiers which have different necessities
2204  *      at different times.
2205  */
2206 int cpufreq_update_policy(unsigned int cpu)
2207 {
2208         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2209         struct cpufreq_policy new_policy;
2210         int ret;
2211
2212         if (!policy)
2213                 return -ENODEV;
2214
2215         down_write(&policy->rwsem);
2216
2217         pr_debug("updating policy for CPU %u\n", cpu);
2218         memcpy(&new_policy, policy, sizeof(*policy));
2219         new_policy.min = policy->user_policy.min;
2220         new_policy.max = policy->user_policy.max;
2221
2222         /*
2223          * BIOS might change freq behind our back
2224          * -> ask driver for current freq and notify governors about a change
2225          */
2226         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2227                 new_policy.cur = cpufreq_driver->get(cpu);
2228                 if (WARN_ON(!new_policy.cur)) {
2229                         ret = -EIO;
2230                         goto unlock;
2231                 }
2232
2233                 if (!policy->cur) {
2234                         pr_debug("Driver did not initialize current freq\n");
2235                         policy->cur = new_policy.cur;
2236                 } else {
2237                         if (policy->cur != new_policy.cur && has_target())
2238                                 cpufreq_out_of_sync(policy, new_policy.cur);
2239                 }
2240         }
2241
2242         ret = cpufreq_set_policy(policy, &new_policy);
2243
2244 unlock:
2245         up_write(&policy->rwsem);
2246
2247         cpufreq_cpu_put(policy);
2248         return ret;
2249 }
2250 EXPORT_SYMBOL(cpufreq_update_policy);
2251
2252 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2253                                         unsigned long action, void *hcpu)
2254 {
2255         unsigned int cpu = (unsigned long)hcpu;
2256
2257         switch (action & ~CPU_TASKS_FROZEN) {
2258         case CPU_ONLINE:
2259                 cpufreq_online(cpu);
2260                 break;
2261
2262         case CPU_DOWN_PREPARE:
2263                 cpufreq_offline_prepare(cpu);
2264                 break;
2265
2266         case CPU_POST_DEAD:
2267                 cpufreq_offline_finish(cpu);
2268                 break;
2269
2270         case CPU_DOWN_FAILED:
2271                 cpufreq_online(cpu);
2272                 break;
2273         }
2274         return NOTIFY_OK;
2275 }
2276
2277 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2278         .notifier_call = cpufreq_cpu_callback,
2279 };
2280
2281 /*********************************************************************
2282  *               BOOST                                               *
2283  *********************************************************************/
2284 static int cpufreq_boost_set_sw(int state)
2285 {
2286         struct cpufreq_frequency_table *freq_table;
2287         struct cpufreq_policy *policy;
2288         int ret = -EINVAL;
2289
2290         for_each_active_policy(policy) {
2291                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2292                 if (freq_table) {
2293                         ret = cpufreq_frequency_table_cpuinfo(policy,
2294                                                         freq_table);
2295                         if (ret) {
2296                                 pr_err("%s: Policy frequency update failed\n",
2297                                        __func__);
2298                                 break;
2299                         }
2300                         policy->user_policy.max = policy->max;
2301                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2302                 }
2303         }
2304
2305         return ret;
2306 }
2307
2308 int cpufreq_boost_trigger_state(int state)
2309 {
2310         unsigned long flags;
2311         int ret = 0;
2312
2313         if (cpufreq_driver->boost_enabled == state)
2314                 return 0;
2315
2316         write_lock_irqsave(&cpufreq_driver_lock, flags);
2317         cpufreq_driver->boost_enabled = state;
2318         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2319
2320         ret = cpufreq_driver->set_boost(state);
2321         if (ret) {
2322                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2323                 cpufreq_driver->boost_enabled = !state;
2324                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2325
2326                 pr_err("%s: Cannot %s BOOST\n",
2327                        __func__, state ? "enable" : "disable");
2328         }
2329
2330         return ret;
2331 }
2332
2333 int cpufreq_boost_supported(void)
2334 {
2335         if (likely(cpufreq_driver))
2336                 return cpufreq_driver->boost_supported;
2337
2338         return 0;
2339 }
2340 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2341
2342 static int create_boost_sysfs_file(void)
2343 {
2344         int ret;
2345
2346         if (!cpufreq_boost_supported())
2347                 return 0;
2348
2349         /*
2350          * Check if driver provides function to enable boost -
2351          * if not, use cpufreq_boost_set_sw as default
2352          */
2353         if (!cpufreq_driver->set_boost)
2354                 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2355
2356         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2357         if (ret)
2358                 pr_err("%s: cannot register global BOOST sysfs file\n",
2359                        __func__);
2360
2361         return ret;
2362 }
2363
2364 static void remove_boost_sysfs_file(void)
2365 {
2366         if (cpufreq_boost_supported())
2367                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2368 }
2369
2370 int cpufreq_enable_boost_support(void)
2371 {
2372         if (!cpufreq_driver)
2373                 return -EINVAL;
2374
2375         if (cpufreq_boost_supported())
2376                 return 0;
2377
2378         cpufreq_driver->boost_supported = true;
2379
2380         /* This will get removed on driver unregister */
2381         return create_boost_sysfs_file();
2382 }
2383 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2384
2385 int cpufreq_boost_enabled(void)
2386 {
2387         return cpufreq_driver->boost_enabled;
2388 }
2389 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2390
2391 /*********************************************************************
2392  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2393  *********************************************************************/
2394
2395 /**
2396  * cpufreq_register_driver - register a CPU Frequency driver
2397  * @driver_data: A struct cpufreq_driver containing the values#
2398  * submitted by the CPU Frequency driver.
2399  *
2400  * Registers a CPU Frequency driver to this core code. This code
2401  * returns zero on success, -EBUSY when another driver got here first
2402  * (and isn't unregistered in the meantime).
2403  *
2404  */
2405 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2406 {
2407         unsigned long flags;
2408         int ret;
2409
2410         if (cpufreq_disabled())
2411                 return -ENODEV;
2412
2413         if (!driver_data || !driver_data->verify || !driver_data->init ||
2414             !(driver_data->setpolicy || driver_data->target_index ||
2415                     driver_data->target) ||
2416              (driver_data->setpolicy && (driver_data->target_index ||
2417                     driver_data->target)) ||
2418              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2419                 return -EINVAL;
2420
2421         pr_debug("trying to register driver %s\n", driver_data->name);
2422
2423         /* Protect against concurrent CPU online/offline. */
2424         get_online_cpus();
2425
2426         write_lock_irqsave(&cpufreq_driver_lock, flags);
2427         if (cpufreq_driver) {
2428                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2429                 ret = -EEXIST;
2430                 goto out;
2431         }
2432         cpufreq_driver = driver_data;
2433         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2434
2435         if (driver_data->setpolicy)
2436                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2437
2438         ret = create_boost_sysfs_file();
2439         if (ret)
2440                 goto err_null_driver;
2441
2442         ret = subsys_interface_register(&cpufreq_interface);
2443         if (ret)
2444                 goto err_boost_unreg;
2445
2446         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2447             list_empty(&cpufreq_policy_list)) {
2448                 /* if all ->init() calls failed, unregister */
2449                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2450                          driver_data->name);
2451                 goto err_if_unreg;
2452         }
2453
2454         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2455         pr_debug("driver %s up and running\n", driver_data->name);
2456
2457 out:
2458         put_online_cpus();
2459         return ret;
2460
2461 err_if_unreg:
2462         subsys_interface_unregister(&cpufreq_interface);
2463 err_boost_unreg:
2464         remove_boost_sysfs_file();
2465 err_null_driver:
2466         write_lock_irqsave(&cpufreq_driver_lock, flags);
2467         cpufreq_driver = NULL;
2468         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2469         goto out;
2470 }
2471 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2472
2473 /**
2474  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2475  *
2476  * Unregister the current CPUFreq driver. Only call this if you have
2477  * the right to do so, i.e. if you have succeeded in initialising before!
2478  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2479  * currently not initialised.
2480  */
2481 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2482 {
2483         unsigned long flags;
2484
2485         if (!cpufreq_driver || (driver != cpufreq_driver))
2486                 return -EINVAL;
2487
2488         pr_debug("unregistering driver %s\n", driver->name);
2489
2490         /* Protect against concurrent cpu hotplug */
2491         get_online_cpus();
2492         subsys_interface_unregister(&cpufreq_interface);
2493         remove_boost_sysfs_file();
2494         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2495
2496         write_lock_irqsave(&cpufreq_driver_lock, flags);
2497
2498         cpufreq_driver = NULL;
2499
2500         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2501         put_online_cpus();
2502
2503         return 0;
2504 }
2505 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2506
2507 /*
2508  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2509  * or mutexes when secondary CPUs are halted.
2510  */
2511 static struct syscore_ops cpufreq_syscore_ops = {
2512         .shutdown = cpufreq_suspend,
2513 };
2514
2515 struct kobject *cpufreq_global_kobject;
2516 EXPORT_SYMBOL(cpufreq_global_kobject);
2517
2518 static int __init cpufreq_core_init(void)
2519 {
2520         if (cpufreq_disabled())
2521                 return -ENODEV;
2522
2523         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2524         BUG_ON(!cpufreq_global_kobject);
2525
2526         register_syscore_ops(&cpufreq_syscore_ops);
2527
2528         return 0;
2529 }
2530 core_initcall(cpufreq_core_init);