2 * x86_pkg_temp_thermal driver
3 * Copyright (c) 2013, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/err.h>
23 #include <linux/param.h>
24 #include <linux/device.h>
25 #include <linux/platform_device.h>
26 #include <linux/cpu.h>
27 #include <linux/smp.h>
28 #include <linux/slab.h>
30 #include <linux/thermal.h>
31 #include <linux/debugfs.h>
32 #include <linux/swork.h>
33 #include <asm/cpu_device_id.h>
37 * Rate control delay: Idea is to introduce denounce effect
38 * This should be long enough to avoid reduce events, when
39 * threshold is set to a temperature, which is constantly
40 * violated, but at the short enough to take any action.
41 * The action can be remove threshold or change it to next
42 * interesting setting. Based on experiments, in around
43 * every 5 seconds under load will give us a significant
46 #define PKG_TEMP_THERMAL_NOTIFY_DELAY 5000
47 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
48 module_param(notify_delay_ms, int, 0644);
49 MODULE_PARM_DESC(notify_delay_ms,
50 "User space notification delay in milli seconds.");
52 /* Number of trip points in thermal zone. Currently it can't
53 * be more than 2. MSR can allow setting and getting notifications
54 * for only 2 thresholds. This define enforces this, if there
55 * is some wrong values returned by cpuid for number of thresholds.
57 #define MAX_NUMBER_OF_TRIPS 2
58 /* Limit number of package temp zones */
59 #define MAX_PKG_TEMP_ZONE_IDS 256
61 struct phy_dev_entry {
62 struct list_head list;
67 u32 start_pkg_therm_low;
68 u32 start_pkg_therm_high;
69 struct thermal_zone_device *tzone;
72 static struct thermal_zone_params pkg_temp_tz_params = {
76 /* List maintaining number of package instances */
77 static LIST_HEAD(phy_dev_list);
78 static DEFINE_MUTEX(phy_dev_list_mutex);
80 /* Interrupt to work function schedule queue */
81 static DEFINE_PER_CPU(struct delayed_work, pkg_temp_thermal_threshold_work);
83 /* To track if the work is already scheduled on a package */
84 static u8 *pkg_work_scheduled;
86 /* Spin lock to prevent races with pkg_work_scheduled */
87 static spinlock_t pkg_work_lock;
88 static u16 max_phy_id;
90 /* Debug counters to show using debugfs */
91 static struct dentry *debugfs;
92 static unsigned int pkg_interrupt_cnt;
93 static unsigned int pkg_work_cnt;
95 static int pkg_temp_debugfs_init(void)
99 debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
103 d = debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
104 (u32 *)&pkg_interrupt_cnt);
108 d = debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
109 (u32 *)&pkg_work_cnt);
116 debugfs_remove_recursive(debugfs);
120 static struct phy_dev_entry
121 *pkg_temp_thermal_get_phy_entry(unsigned int cpu)
123 u16 phys_proc_id = topology_physical_package_id(cpu);
124 struct phy_dev_entry *phy_ptr;
126 mutex_lock(&phy_dev_list_mutex);
128 list_for_each_entry(phy_ptr, &phy_dev_list, list)
129 if (phy_ptr->phys_proc_id == phys_proc_id) {
130 mutex_unlock(&phy_dev_list_mutex);
134 mutex_unlock(&phy_dev_list_mutex);
140 * tj-max is is interesting because threshold is set relative to this
143 static int get_tj_max(int cpu, u32 *tj_max)
149 err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
153 val = (eax >> 16) & 0xff;
155 *tj_max = val * 1000;
168 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
171 struct phy_dev_entry *phy_dev_entry;
173 phy_dev_entry = tzd->devdata;
174 rdmsr_on_cpu(phy_dev_entry->first_cpu, MSR_IA32_PACKAGE_THERM_STATUS,
176 if (eax & 0x80000000) {
177 *temp = phy_dev_entry->tj_max -
178 ((eax >> 16) & 0x7f) * 1000;
179 pr_debug("sys_get_curr_temp %d\n", *temp);
186 static int sys_get_trip_temp(struct thermal_zone_device *tzd,
190 struct phy_dev_entry *phy_dev_entry;
192 unsigned long thres_reg_value;
195 if (trip >= MAX_NUMBER_OF_TRIPS)
198 phy_dev_entry = tzd->devdata;
201 mask = THERM_MASK_THRESHOLD1;
202 shift = THERM_SHIFT_THRESHOLD1;
204 mask = THERM_MASK_THRESHOLD0;
205 shift = THERM_SHIFT_THRESHOLD0;
208 ret = rdmsr_on_cpu(phy_dev_entry->first_cpu,
209 MSR_IA32_PACKAGE_THERM_INTERRUPT, &eax, &edx);
213 thres_reg_value = (eax & mask) >> shift;
215 *temp = phy_dev_entry->tj_max - thres_reg_value * 1000;
218 pr_debug("sys_get_trip_temp %d\n", *temp);
223 static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
227 struct phy_dev_entry *phy_dev_entry;
228 u32 mask, shift, intr;
231 phy_dev_entry = tzd->devdata;
233 if (trip >= MAX_NUMBER_OF_TRIPS || temp >= phy_dev_entry->tj_max)
236 ret = rdmsr_on_cpu(phy_dev_entry->first_cpu,
237 MSR_IA32_PACKAGE_THERM_INTERRUPT,
243 mask = THERM_MASK_THRESHOLD1;
244 shift = THERM_SHIFT_THRESHOLD1;
245 intr = THERM_INT_THRESHOLD1_ENABLE;
247 mask = THERM_MASK_THRESHOLD0;
248 shift = THERM_SHIFT_THRESHOLD0;
249 intr = THERM_INT_THRESHOLD0_ENABLE;
253 * When users space sets a trip temperature == 0, which is indication
254 * that, it is no longer interested in receiving notifications.
259 l |= (phy_dev_entry->tj_max - temp)/1000 << shift;
263 return wrmsr_on_cpu(phy_dev_entry->first_cpu,
264 MSR_IA32_PACKAGE_THERM_INTERRUPT,
268 static int sys_get_trip_type(struct thermal_zone_device *thermal,
269 int trip, enum thermal_trip_type *type)
272 *type = THERMAL_TRIP_PASSIVE;
277 /* Thermal zone callback registry */
278 static struct thermal_zone_device_ops tzone_ops = {
279 .get_temp = sys_get_curr_temp,
280 .get_trip_temp = sys_get_trip_temp,
281 .get_trip_type = sys_get_trip_type,
282 .set_trip_temp = sys_set_trip_temp,
285 static bool pkg_temp_thermal_platform_thermal_rate_control(void)
290 /* Enable threshold interrupt on local package/cpu */
291 static inline void enable_pkg_thres_interrupt(void)
296 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
297 /* only enable/disable if it had valid threshold value */
298 thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
299 thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
301 l |= THERM_INT_THRESHOLD0_ENABLE;
303 l |= THERM_INT_THRESHOLD1_ENABLE;
304 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
307 /* Disable threshold interrupt on local package/cpu */
308 static inline void disable_pkg_thres_interrupt(void)
311 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
312 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
313 l & (~THERM_INT_THRESHOLD0_ENABLE) &
314 (~THERM_INT_THRESHOLD1_ENABLE), h);
317 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
320 int cpu = smp_processor_id();
321 int phy_id = topology_physical_package_id(cpu);
322 struct phy_dev_entry *phdev = pkg_temp_thermal_get_phy_entry(cpu);
329 spin_lock_irqsave(&pkg_work_lock, flags);
331 if (unlikely(phy_id > max_phy_id)) {
332 spin_unlock_irqrestore(&pkg_work_lock, flags);
335 pkg_work_scheduled[phy_id] = 0;
336 spin_unlock_irqrestore(&pkg_work_lock, flags);
338 enable_pkg_thres_interrupt();
339 rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
340 if (msr_val & THERM_LOG_THRESHOLD0) {
341 wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS,
342 msr_val & ~THERM_LOG_THRESHOLD0);
345 if (msr_val & THERM_LOG_THRESHOLD1) {
346 wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS,
347 msr_val & ~THERM_LOG_THRESHOLD1);
351 pr_debug("thermal_zone_device_update\n");
352 thermal_zone_device_update(phdev->tzone);
356 static void platform_thermal_notify_work(struct swork_event *event)
359 int cpu = smp_processor_id();
360 int phy_id = topology_physical_package_id(cpu);
363 * When a package is in interrupted state, all CPU's in that package
364 * are in the same interrupt state. So scheduling on any one CPU in
365 * the package is enough and simply return for others.
367 spin_lock_irqsave(&pkg_work_lock, flags);
369 if (unlikely(phy_id > max_phy_id) || unlikely(!pkg_work_scheduled) ||
370 pkg_work_scheduled[phy_id]) {
371 disable_pkg_thres_interrupt();
372 spin_unlock_irqrestore(&pkg_work_lock, flags);
375 pkg_work_scheduled[phy_id] = 1;
376 spin_unlock_irqrestore(&pkg_work_lock, flags);
378 disable_pkg_thres_interrupt();
379 schedule_delayed_work_on(cpu,
380 &per_cpu(pkg_temp_thermal_threshold_work, cpu),
381 msecs_to_jiffies(notify_delay_ms));
384 #ifdef CONFIG_PREEMPT_RT_FULL
385 static struct swork_event notify_work;
387 static int thermal_notify_work_init(void)
395 INIT_SWORK(¬ify_work, platform_thermal_notify_work);
399 static void thermal_notify_work_cleanup(void)
404 static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
406 swork_queue(¬ify_work);
410 #else /* !CONFIG_PREEMPT_RT_FULL */
412 static int thermal_notify_work_init(void) { return 0; }
414 static void thermal_notify_work_cleanup(void) { }
416 static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
418 platform_thermal_notify_work(NULL);
422 #endif /* CONFIG_PREEMPT_RT_FULL */
424 static int find_siblings_cpu(int cpu)
427 int id = topology_physical_package_id(cpu);
429 for_each_online_cpu(i)
430 if (i != cpu && topology_physical_package_id(i) == id)
436 static int pkg_temp_thermal_device_add(unsigned int cpu)
440 struct phy_dev_entry *phy_dev_entry;
442 u32 eax, ebx, ecx, edx;
446 cpuid(6, &eax, &ebx, &ecx, &edx);
447 thres_count = ebx & 0x07;
451 if (topology_physical_package_id(cpu) > MAX_PKG_TEMP_ZONE_IDS)
454 thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
456 err = get_tj_max(cpu, &tj_max);
460 mutex_lock(&phy_dev_list_mutex);
462 phy_dev_entry = kzalloc(sizeof(*phy_dev_entry), GFP_KERNEL);
463 if (!phy_dev_entry) {
468 spin_lock_irqsave(&pkg_work_lock, flags);
469 if (topology_physical_package_id(cpu) > max_phy_id)
470 max_phy_id = topology_physical_package_id(cpu);
471 temp = krealloc(pkg_work_scheduled,
472 (max_phy_id+1) * sizeof(u8), GFP_ATOMIC);
474 spin_unlock_irqrestore(&pkg_work_lock, flags);
478 pkg_work_scheduled = temp;
479 pkg_work_scheduled[topology_physical_package_id(cpu)] = 0;
480 spin_unlock_irqrestore(&pkg_work_lock, flags);
482 phy_dev_entry->phys_proc_id = topology_physical_package_id(cpu);
483 phy_dev_entry->first_cpu = cpu;
484 phy_dev_entry->tj_max = tj_max;
485 phy_dev_entry->ref_cnt = 1;
486 phy_dev_entry->tzone = thermal_zone_device_register("x86_pkg_temp",
488 (thres_count == MAX_NUMBER_OF_TRIPS) ?
490 phy_dev_entry, &tzone_ops, &pkg_temp_tz_params, 0, 0);
491 if (IS_ERR(phy_dev_entry->tzone)) {
492 err = PTR_ERR(phy_dev_entry->tzone);
495 /* Store MSR value for package thermal interrupt, to restore at exit */
496 rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
497 &phy_dev_entry->start_pkg_therm_low,
498 &phy_dev_entry->start_pkg_therm_high);
500 list_add_tail(&phy_dev_entry->list, &phy_dev_list);
501 pr_debug("pkg_temp_thermal_device_add :phy_id %d cpu %d\n",
502 phy_dev_entry->phys_proc_id, cpu);
504 mutex_unlock(&phy_dev_list_mutex);
509 kfree(phy_dev_entry);
511 mutex_unlock(&phy_dev_list_mutex);
517 static int pkg_temp_thermal_device_remove(unsigned int cpu)
519 struct phy_dev_entry *n;
520 u16 phys_proc_id = topology_physical_package_id(cpu);
521 struct phy_dev_entry *phdev =
522 pkg_temp_thermal_get_phy_entry(cpu);
527 mutex_lock(&phy_dev_list_mutex);
528 /* If we are loosing the first cpu for this package, we need change */
529 if (phdev->first_cpu == cpu) {
530 phdev->first_cpu = find_siblings_cpu(cpu);
531 pr_debug("thermal_device_remove: first cpu switched %d\n",
535 * It is possible that no siblings left as this was the last cpu
536 * going offline. We don't need to worry about this assignment
537 * as the phydev entry will be removed in this case and
538 * thermal zone is removed.
541 pr_debug("thermal_device_remove: pkg: %d cpu %d ref_cnt %d\n",
542 phys_proc_id, cpu, phdev->ref_cnt);
544 list_for_each_entry_safe(phdev, n, &phy_dev_list, list) {
545 if (phdev->phys_proc_id == phys_proc_id) {
546 thermal_zone_device_unregister(phdev->tzone);
547 list_del(&phdev->list);
552 mutex_unlock(&phy_dev_list_mutex);
557 static int get_core_online(unsigned int cpu)
559 struct cpuinfo_x86 *c = &cpu_data(cpu);
560 struct phy_dev_entry *phdev = pkg_temp_thermal_get_phy_entry(cpu);
562 /* Check if there is already an instance for this package */
564 if (!cpu_has(c, X86_FEATURE_DTHERM) ||
565 !cpu_has(c, X86_FEATURE_PTS))
567 if (pkg_temp_thermal_device_add(cpu))
570 mutex_lock(&phy_dev_list_mutex);
572 pr_debug("get_core_online: cpu %d ref_cnt %d\n",
573 cpu, phdev->ref_cnt);
574 mutex_unlock(&phy_dev_list_mutex);
576 INIT_DELAYED_WORK(&per_cpu(pkg_temp_thermal_threshold_work, cpu),
577 pkg_temp_thermal_threshold_work_fn);
579 pr_debug("get_core_online: cpu %d successful\n", cpu);
584 static void put_core_offline(unsigned int cpu)
586 if (!pkg_temp_thermal_device_remove(cpu))
587 cancel_delayed_work_sync(
588 &per_cpu(pkg_temp_thermal_threshold_work, cpu));
590 pr_debug("put_core_offline: cpu %d\n", cpu);
593 static int pkg_temp_thermal_cpu_callback(struct notifier_block *nfb,
594 unsigned long action, void *hcpu)
596 unsigned int cpu = (unsigned long) hcpu;
600 case CPU_DOWN_FAILED:
601 get_core_online(cpu);
603 case CPU_DOWN_PREPARE:
604 put_core_offline(cpu);
610 static struct notifier_block pkg_temp_thermal_notifier __refdata = {
611 .notifier_call = pkg_temp_thermal_cpu_callback,
614 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
615 { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_PTS },
618 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
620 static int __init pkg_temp_thermal_init(void)
624 if (!x86_match_cpu(pkg_temp_thermal_ids))
627 if (!thermal_notify_work_init())
630 spin_lock_init(&pkg_work_lock);
631 platform_thermal_package_notify =
632 pkg_temp_thermal_platform_thermal_notify;
633 platform_thermal_package_rate_control =
634 pkg_temp_thermal_platform_thermal_rate_control;
636 cpu_notifier_register_begin();
637 for_each_online_cpu(i)
638 if (get_core_online(i))
640 __register_hotcpu_notifier(&pkg_temp_thermal_notifier);
641 cpu_notifier_register_done();
643 pkg_temp_debugfs_init(); /* Don't care if fails */
648 for_each_online_cpu(i)
650 cpu_notifier_register_done();
651 kfree(pkg_work_scheduled);
652 platform_thermal_package_notify = NULL;
653 platform_thermal_package_rate_control = NULL;
654 thermal_notify_work_cleanup();
658 static void __exit pkg_temp_thermal_exit(void)
660 struct phy_dev_entry *phdev, *n;
663 cpu_notifier_register_begin();
664 __unregister_hotcpu_notifier(&pkg_temp_thermal_notifier);
665 mutex_lock(&phy_dev_list_mutex);
666 list_for_each_entry_safe(phdev, n, &phy_dev_list, list) {
667 /* Retore old MSR value for package thermal interrupt */
668 wrmsr_on_cpu(phdev->first_cpu,
669 MSR_IA32_PACKAGE_THERM_INTERRUPT,
670 phdev->start_pkg_therm_low,
671 phdev->start_pkg_therm_high);
672 thermal_zone_device_unregister(phdev->tzone);
673 list_del(&phdev->list);
676 mutex_unlock(&phy_dev_list_mutex);
677 platform_thermal_package_notify = NULL;
678 platform_thermal_package_rate_control = NULL;
679 thermal_notify_work_cleanup();
680 for_each_online_cpu(i)
681 cancel_delayed_work_sync(
682 &per_cpu(pkg_temp_thermal_threshold_work, i));
683 cpu_notifier_register_done();
685 kfree(pkg_work_scheduled);
687 debugfs_remove_recursive(debugfs);
690 module_init(pkg_temp_thermal_init)
691 module_exit(pkg_temp_thermal_exit)
693 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
694 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
695 MODULE_LICENSE("GPL v2");