2 * Intel Cache Quality-of-Service Monitoring (CQM) support.
4 * Based very, very heavily on work by Peter Zijlstra.
7 #include <linux/perf_event.h>
8 #include <linux/slab.h>
9 #include <asm/cpu_device_id.h>
10 #include "perf_event.h"
12 #define MSR_IA32_PQR_ASSOC 0x0c8f
13 #define MSR_IA32_QM_CTR 0x0c8e
14 #define MSR_IA32_QM_EVTSEL 0x0c8d
16 static u32 cqm_max_rmid = -1;
17 static unsigned int cqm_l3_scale; /* supposedly cacheline size */
20 * struct intel_pqr_state - State cache for the PQR MSR
21 * @rmid: The cached Resource Monitoring ID
22 * @closid: The cached Class Of Service ID
23 * @rmid_usecnt: The usage counter for rmid
25 * The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
26 * lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
27 * contains both parts, so we need to cache them.
29 * The cache also helps to avoid pointless updates if the value does
32 struct intel_pqr_state {
39 * The cached intel_pqr_state is strictly per CPU and can never be
40 * updated from a remote CPU. Both functions which modify the state
41 * (intel_cqm_event_start and intel_cqm_event_stop) are called with
42 * interrupts disabled, which is sufficient for the protection.
44 static DEFINE_PER_CPU(struct intel_pqr_state, pqr_state);
47 * Protects cache_cgroups and cqm_rmid_free_lru and cqm_rmid_limbo_lru.
48 * Also protects event->hw.cqm_rmid
50 * Hold either for stability, both for modification of ->hw.cqm_rmid.
52 static DEFINE_MUTEX(cache_mutex);
53 static DEFINE_RAW_SPINLOCK(cache_lock);
56 * Groups of events that have the same target(s), one RMID per group.
58 static LIST_HEAD(cache_groups);
61 * Mask of CPUs for reading CQM values. We only need one per-socket.
63 static cpumask_t cqm_cpumask;
65 #define RMID_VAL_ERROR (1ULL << 63)
66 #define RMID_VAL_UNAVAIL (1ULL << 62)
68 #define QOS_L3_OCCUP_EVENT_ID (1 << 0)
70 #define QOS_EVENT_MASK QOS_L3_OCCUP_EVENT_ID
73 * This is central to the rotation algorithm in __intel_cqm_rmid_rotate().
75 * This rmid is always free and is guaranteed to have an associated
76 * near-zero occupancy value, i.e. no cachelines are tagged with this
77 * RMID, once __intel_cqm_rmid_rotate() returns.
79 static u32 intel_cqm_rotation_rmid;
81 #define INVALID_RMID (-1)
84 * Is @rmid valid for programming the hardware?
86 * rmid 0 is reserved by the hardware for all non-monitored tasks, which
87 * means that we should never come across an rmid with that value.
88 * Likewise, an rmid value of -1 is used to indicate "no rmid currently
89 * assigned" and is used as part of the rotation code.
91 static inline bool __rmid_valid(u32 rmid)
93 if (!rmid || rmid == INVALID_RMID)
99 static u64 __rmid_read(u32 rmid)
104 * Ignore the SDM, this thing is _NOTHING_ like a regular perfcnt,
105 * it just says that to increase confusion.
107 wrmsr(MSR_IA32_QM_EVTSEL, QOS_L3_OCCUP_EVENT_ID, rmid);
108 rdmsrl(MSR_IA32_QM_CTR, val);
111 * Aside from the ERROR and UNAVAIL bits, assume this thing returns
112 * the number of cachelines tagged with @rmid.
117 enum rmid_recycle_state {
123 struct cqm_rmid_entry {
125 enum rmid_recycle_state state;
126 struct list_head list;
127 unsigned long queue_time;
131 * cqm_rmid_free_lru - A least recently used list of RMIDs.
133 * Oldest entry at the head, newest (most recently used) entry at the
134 * tail. This list is never traversed, it's only used to keep track of
135 * the lru order. That is, we only pick entries of the head or insert
138 * All entries on the list are 'free', and their RMIDs are not currently
139 * in use. To mark an RMID as in use, remove its entry from the lru
143 * cqm_rmid_limbo_lru - list of currently unused but (potentially) dirty RMIDs.
145 * This list is contains RMIDs that no one is currently using but that
146 * may have a non-zero occupancy value associated with them. The
147 * rotation worker moves RMIDs from the limbo list to the free list once
148 * the occupancy value drops below __intel_cqm_threshold.
150 * Both lists are protected by cache_mutex.
152 static LIST_HEAD(cqm_rmid_free_lru);
153 static LIST_HEAD(cqm_rmid_limbo_lru);
156 * We use a simple array of pointers so that we can lookup a struct
157 * cqm_rmid_entry in O(1). This alleviates the callers of __get_rmid()
158 * and __put_rmid() from having to worry about dealing with struct
159 * cqm_rmid_entry - they just deal with rmids, i.e. integers.
161 * Once this array is initialized it is read-only. No locks are required
164 * All entries for all RMIDs can be looked up in the this array at all
167 static struct cqm_rmid_entry **cqm_rmid_ptrs;
169 static inline struct cqm_rmid_entry *__rmid_entry(u32 rmid)
171 struct cqm_rmid_entry *entry;
173 entry = cqm_rmid_ptrs[rmid];
174 WARN_ON(entry->rmid != rmid);
180 * Returns < 0 on fail.
182 * We expect to be called with cache_mutex held.
184 static u32 __get_rmid(void)
186 struct cqm_rmid_entry *entry;
188 lockdep_assert_held(&cache_mutex);
190 if (list_empty(&cqm_rmid_free_lru))
193 entry = list_first_entry(&cqm_rmid_free_lru, struct cqm_rmid_entry, list);
194 list_del(&entry->list);
199 static void __put_rmid(u32 rmid)
201 struct cqm_rmid_entry *entry;
203 lockdep_assert_held(&cache_mutex);
205 WARN_ON(!__rmid_valid(rmid));
206 entry = __rmid_entry(rmid);
208 entry->queue_time = jiffies;
209 entry->state = RMID_YOUNG;
211 list_add_tail(&entry->list, &cqm_rmid_limbo_lru);
214 static void cqm_cleanup(void)
221 for (i = 0; i < cqm_max_rmid; i++)
222 kfree(cqm_rmid_ptrs[i]);
224 kfree(cqm_rmid_ptrs);
225 cqm_rmid_ptrs = NULL;
228 static int intel_cqm_setup_rmid_cache(void)
230 struct cqm_rmid_entry *entry;
231 unsigned int nr_rmids;
234 nr_rmids = cqm_max_rmid + 1;
235 cqm_rmid_ptrs = kzalloc(sizeof(struct cqm_rmid_entry *) *
236 nr_rmids, GFP_KERNEL);
240 for (; r <= cqm_max_rmid; r++) {
241 struct cqm_rmid_entry *entry;
243 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
247 INIT_LIST_HEAD(&entry->list);
249 cqm_rmid_ptrs[r] = entry;
251 list_add_tail(&entry->list, &cqm_rmid_free_lru);
255 * RMID 0 is special and is always allocated. It's used for all
256 * tasks that are not monitored.
258 entry = __rmid_entry(0);
259 list_del(&entry->list);
261 mutex_lock(&cache_mutex);
262 intel_cqm_rotation_rmid = __get_rmid();
263 mutex_unlock(&cache_mutex);
273 * Determine if @a and @b measure the same set of tasks.
275 * If @a and @b measure the same set of tasks then we want to share a
278 static bool __match_event(struct perf_event *a, struct perf_event *b)
280 /* Per-cpu and task events don't mix */
281 if ((a->attach_state & PERF_ATTACH_TASK) !=
282 (b->attach_state & PERF_ATTACH_TASK))
285 #ifdef CONFIG_CGROUP_PERF
286 if (a->cgrp != b->cgrp)
290 /* If not task event, we're machine wide */
291 if (!(b->attach_state & PERF_ATTACH_TASK))
295 * Events that target same task are placed into the same cache group.
296 * Mark it as a multi event group, so that we update ->count
297 * for every event rather than just the group leader later.
299 if (a->hw.target == b->hw.target) {
300 b->hw.is_group_event = true;
305 * Are we an inherited event?
313 #ifdef CONFIG_CGROUP_PERF
314 static inline struct perf_cgroup *event_to_cgroup(struct perf_event *event)
316 if (event->attach_state & PERF_ATTACH_TASK)
317 return perf_cgroup_from_task(event->hw.target, event->ctx);
324 * Determine if @a's tasks intersect with @b's tasks
326 * There are combinations of events that we explicitly prohibit,
329 * system-wide -> cgroup and task
330 * cgroup -> system-wide
332 * task -> system-wide
335 * Call this function before allocating an RMID.
337 static bool __conflict_event(struct perf_event *a, struct perf_event *b)
339 #ifdef CONFIG_CGROUP_PERF
341 * We can have any number of cgroups but only one system-wide
344 if (a->cgrp && b->cgrp) {
345 struct perf_cgroup *ac = a->cgrp;
346 struct perf_cgroup *bc = b->cgrp;
349 * This condition should have been caught in
350 * __match_event() and we should be sharing an RMID.
352 WARN_ON_ONCE(ac == bc);
354 if (cgroup_is_descendant(ac->css.cgroup, bc->css.cgroup) ||
355 cgroup_is_descendant(bc->css.cgroup, ac->css.cgroup))
361 if (a->cgrp || b->cgrp) {
362 struct perf_cgroup *ac, *bc;
365 * cgroup and system-wide events are mutually exclusive
367 if ((a->cgrp && !(b->attach_state & PERF_ATTACH_TASK)) ||
368 (b->cgrp && !(a->attach_state & PERF_ATTACH_TASK)))
372 * Ensure neither event is part of the other's cgroup
374 ac = event_to_cgroup(a);
375 bc = event_to_cgroup(b);
380 * Must have cgroup and non-intersecting task events.
386 * We have cgroup and task events, and the task belongs
387 * to a cgroup. Check for for overlap.
389 if (cgroup_is_descendant(ac->css.cgroup, bc->css.cgroup) ||
390 cgroup_is_descendant(bc->css.cgroup, ac->css.cgroup))
397 * If one of them is not a task, same story as above with cgroups.
399 if (!(a->attach_state & PERF_ATTACH_TASK) ||
400 !(b->attach_state & PERF_ATTACH_TASK))
404 * Must be non-overlapping.
414 static void __intel_cqm_event_count(void *info);
417 * Exchange the RMID of a group of events.
419 static u32 intel_cqm_xchg_rmid(struct perf_event *group, u32 rmid)
421 struct perf_event *event;
422 struct list_head *head = &group->hw.cqm_group_entry;
423 u32 old_rmid = group->hw.cqm_rmid;
425 lockdep_assert_held(&cache_mutex);
428 * If our RMID is being deallocated, perform a read now.
430 if (__rmid_valid(old_rmid) && !__rmid_valid(rmid)) {
431 struct rmid_read rr = {
432 .value = ATOMIC64_INIT(0),
436 on_each_cpu_mask(&cqm_cpumask, __intel_cqm_event_count,
438 local64_set(&group->count, atomic64_read(&rr.value));
441 raw_spin_lock_irq(&cache_lock);
443 group->hw.cqm_rmid = rmid;
444 list_for_each_entry(event, head, hw.cqm_group_entry)
445 event->hw.cqm_rmid = rmid;
447 raw_spin_unlock_irq(&cache_lock);
453 * If we fail to assign a new RMID for intel_cqm_rotation_rmid because
454 * cachelines are still tagged with RMIDs in limbo, we progressively
455 * increment the threshold until we find an RMID in limbo with <=
456 * __intel_cqm_threshold lines tagged. This is designed to mitigate the
457 * problem where cachelines tagged with an RMID are not steadily being
460 * On successful rotations we decrease the threshold back towards zero.
462 * __intel_cqm_max_threshold provides an upper bound on the threshold,
463 * and is measured in bytes because it's exposed to userland.
465 static unsigned int __intel_cqm_threshold;
466 static unsigned int __intel_cqm_max_threshold;
469 * Test whether an RMID has a zero occupancy value on this cpu.
471 static void intel_cqm_stable(void *arg)
473 struct cqm_rmid_entry *entry;
475 list_for_each_entry(entry, &cqm_rmid_limbo_lru, list) {
476 if (entry->state != RMID_AVAILABLE)
479 if (__rmid_read(entry->rmid) > __intel_cqm_threshold)
480 entry->state = RMID_DIRTY;
485 * If we have group events waiting for an RMID that don't conflict with
486 * events already running, assign @rmid.
488 static bool intel_cqm_sched_in_event(u32 rmid)
490 struct perf_event *leader, *event;
492 lockdep_assert_held(&cache_mutex);
494 leader = list_first_entry(&cache_groups, struct perf_event,
495 hw.cqm_groups_entry);
498 list_for_each_entry_continue(event, &cache_groups,
499 hw.cqm_groups_entry) {
500 if (__rmid_valid(event->hw.cqm_rmid))
503 if (__conflict_event(event, leader))
506 intel_cqm_xchg_rmid(event, rmid);
514 * Initially use this constant for both the limbo queue time and the
515 * rotation timer interval, pmu::hrtimer_interval_ms.
517 * They don't need to be the same, but the two are related since if you
518 * rotate faster than you recycle RMIDs, you may run out of available
521 #define RMID_DEFAULT_QUEUE_TIME 250 /* ms */
523 static unsigned int __rmid_queue_time_ms = RMID_DEFAULT_QUEUE_TIME;
526 * intel_cqm_rmid_stabilize - move RMIDs from limbo to free list
527 * @nr_available: number of freeable RMIDs on the limbo list
529 * Quiescent state; wait for all 'freed' RMIDs to become unused, i.e. no
530 * cachelines are tagged with those RMIDs. After this we can reuse them
531 * and know that the current set of active RMIDs is stable.
533 * Return %true or %false depending on whether stabilization needs to be
536 * If we return %true then @nr_available is updated to indicate the
537 * number of RMIDs on the limbo list that have been queued for the
538 * minimum queue time (RMID_AVAILABLE), but whose data occupancy values
539 * are above __intel_cqm_threshold.
541 static bool intel_cqm_rmid_stabilize(unsigned int *available)
543 struct cqm_rmid_entry *entry, *tmp;
545 lockdep_assert_held(&cache_mutex);
548 list_for_each_entry(entry, &cqm_rmid_limbo_lru, list) {
549 unsigned long min_queue_time;
550 unsigned long now = jiffies;
553 * We hold RMIDs placed into limbo for a minimum queue
554 * time. Before the minimum queue time has elapsed we do
557 * The reasoning is that until a sufficient time has
558 * passed since we stopped using an RMID, any RMID
559 * placed onto the limbo list will likely still have
560 * data tagged in the cache, which means we'll probably
561 * fail to recycle it anyway.
563 * We can save ourselves an expensive IPI by skipping
564 * any RMIDs that have not been queued for the minimum
567 min_queue_time = entry->queue_time +
568 msecs_to_jiffies(__rmid_queue_time_ms);
570 if (time_after(min_queue_time, now))
573 entry->state = RMID_AVAILABLE;
578 * Fast return if none of the RMIDs on the limbo list have been
579 * sitting on the queue for the minimum queue time.
585 * Test whether an RMID is free for each package.
587 on_each_cpu_mask(&cqm_cpumask, intel_cqm_stable, NULL, true);
589 list_for_each_entry_safe(entry, tmp, &cqm_rmid_limbo_lru, list) {
591 * Exhausted all RMIDs that have waited min queue time.
593 if (entry->state == RMID_YOUNG)
596 if (entry->state == RMID_DIRTY)
599 list_del(&entry->list); /* remove from limbo */
602 * The rotation RMID gets priority if it's
603 * currently invalid. In which case, skip adding
604 * the RMID to the the free lru.
606 if (!__rmid_valid(intel_cqm_rotation_rmid)) {
607 intel_cqm_rotation_rmid = entry->rmid;
612 * If we have groups waiting for RMIDs, hand
613 * them one now provided they don't conflict.
615 if (intel_cqm_sched_in_event(entry->rmid))
619 * Otherwise place it onto the free list.
621 list_add_tail(&entry->list, &cqm_rmid_free_lru);
625 return __rmid_valid(intel_cqm_rotation_rmid);
629 * Pick a victim group and move it to the tail of the group list.
630 * @next: The first group without an RMID
632 static void __intel_cqm_pick_and_rotate(struct perf_event *next)
634 struct perf_event *rotor;
637 lockdep_assert_held(&cache_mutex);
639 rotor = list_first_entry(&cache_groups, struct perf_event,
640 hw.cqm_groups_entry);
643 * The group at the front of the list should always have a valid
644 * RMID. If it doesn't then no groups have RMIDs assigned and we
645 * don't need to rotate the list.
650 rmid = intel_cqm_xchg_rmid(rotor, INVALID_RMID);
653 list_rotate_left(&cache_groups);
657 * Deallocate the RMIDs from any events that conflict with @event, and
658 * place them on the back of the group list.
660 static void intel_cqm_sched_out_conflicting_events(struct perf_event *event)
662 struct perf_event *group, *g;
665 lockdep_assert_held(&cache_mutex);
667 list_for_each_entry_safe(group, g, &cache_groups, hw.cqm_groups_entry) {
671 rmid = group->hw.cqm_rmid;
674 * Skip events that don't have a valid RMID.
676 if (!__rmid_valid(rmid))
680 * No conflict? No problem! Leave the event alone.
682 if (!__conflict_event(group, event))
685 intel_cqm_xchg_rmid(group, INVALID_RMID);
691 * Attempt to rotate the groups and assign new RMIDs.
693 * We rotate for two reasons,
694 * 1. To handle the scheduling of conflicting events
695 * 2. To recycle RMIDs
697 * Rotating RMIDs is complicated because the hardware doesn't give us
700 * There's problems with the hardware interface; when you change the
701 * task:RMID map cachelines retain their 'old' tags, giving a skewed
702 * picture. In order to work around this, we must always keep one free
703 * RMID - intel_cqm_rotation_rmid.
705 * Rotation works by taking away an RMID from a group (the old RMID),
706 * and assigning the free RMID to another group (the new RMID). We must
707 * then wait for the old RMID to not be used (no cachelines tagged).
708 * This ensure that all cachelines are tagged with 'active' RMIDs. At
709 * this point we can start reading values for the new RMID and treat the
710 * old RMID as the free RMID for the next rotation.
712 * Return %true or %false depending on whether we did any rotating.
714 static bool __intel_cqm_rmid_rotate(void)
716 struct perf_event *group, *start = NULL;
717 unsigned int threshold_limit;
718 unsigned int nr_needed = 0;
719 unsigned int nr_available;
720 bool rotated = false;
722 mutex_lock(&cache_mutex);
726 * Fast path through this function if there are no groups and no
727 * RMIDs that need cleaning.
729 if (list_empty(&cache_groups) && list_empty(&cqm_rmid_limbo_lru))
732 list_for_each_entry(group, &cache_groups, hw.cqm_groups_entry) {
733 if (!__rmid_valid(group->hw.cqm_rmid)) {
741 * We have some event groups, but they all have RMIDs assigned
742 * and no RMIDs need cleaning.
744 if (!nr_needed && list_empty(&cqm_rmid_limbo_lru))
751 * We have more event groups without RMIDs than available RMIDs,
752 * or we have event groups that conflict with the ones currently
755 * We force deallocate the rmid of the group at the head of
756 * cache_groups. The first event group without an RMID then gets
757 * assigned intel_cqm_rotation_rmid. This ensures we always make
760 * Rotate the cache_groups list so the previous head is now the
763 __intel_cqm_pick_and_rotate(start);
766 * If the rotation is going to succeed, reduce the threshold so
767 * that we don't needlessly reuse dirty RMIDs.
769 if (__rmid_valid(intel_cqm_rotation_rmid)) {
770 intel_cqm_xchg_rmid(start, intel_cqm_rotation_rmid);
771 intel_cqm_rotation_rmid = __get_rmid();
773 intel_cqm_sched_out_conflicting_events(start);
775 if (__intel_cqm_threshold)
776 __intel_cqm_threshold--;
783 * We now need to stablize the RMID we freed above (if any) to
784 * ensure that the next time we rotate we have an RMID with zero
787 * Alternatively, if we didn't need to perform any rotation,
788 * we'll have a bunch of RMIDs in limbo that need stabilizing.
790 threshold_limit = __intel_cqm_max_threshold / cqm_l3_scale;
792 while (intel_cqm_rmid_stabilize(&nr_available) &&
793 __intel_cqm_threshold < threshold_limit) {
794 unsigned int steal_limit;
797 * Don't spin if nobody is actively waiting for an RMID,
798 * the rotation worker will be kicked as soon as an
799 * event needs an RMID anyway.
804 /* Allow max 25% of RMIDs to be in limbo. */
805 steal_limit = (cqm_max_rmid + 1) / 4;
808 * We failed to stabilize any RMIDs so our rotation
809 * logic is now stuck. In order to make forward progress
810 * we have a few options:
812 * 1. rotate ("steal") another RMID
813 * 2. increase the threshold
816 * We do both of 1. and 2. until we hit the steal limit.
818 * The steal limit prevents all RMIDs ending up on the
819 * limbo list. This can happen if every RMID has a
820 * non-zero occupancy above threshold_limit, and the
821 * occupancy values aren't dropping fast enough.
823 * Note that there is prioritisation at work here - we'd
824 * rather increase the number of RMIDs on the limbo list
825 * than increase the threshold, because increasing the
826 * threshold skews the event data (because we reuse
827 * dirty RMIDs) - threshold bumps are a last resort.
829 if (nr_available < steal_limit)
832 __intel_cqm_threshold++;
836 mutex_unlock(&cache_mutex);
840 static void intel_cqm_rmid_rotate(struct work_struct *work);
842 static DECLARE_DELAYED_WORK(intel_cqm_rmid_work, intel_cqm_rmid_rotate);
844 static struct pmu intel_cqm_pmu;
846 static void intel_cqm_rmid_rotate(struct work_struct *work)
850 __intel_cqm_rmid_rotate();
852 delay = msecs_to_jiffies(intel_cqm_pmu.hrtimer_interval_ms);
853 schedule_delayed_work(&intel_cqm_rmid_work, delay);
857 * Find a group and setup RMID.
859 * If we're part of a group, we use the group's RMID.
861 static void intel_cqm_setup_event(struct perf_event *event,
862 struct perf_event **group)
864 struct perf_event *iter;
865 bool conflict = false;
868 event->hw.is_group_event = false;
869 list_for_each_entry(iter, &cache_groups, hw.cqm_groups_entry) {
870 rmid = iter->hw.cqm_rmid;
872 if (__match_event(iter, event)) {
873 /* All tasks in a group share an RMID */
874 event->hw.cqm_rmid = rmid;
880 * We only care about conflicts for events that are
881 * actually scheduled in (and hence have a valid RMID).
883 if (__conflict_event(iter, event) && __rmid_valid(rmid))
892 event->hw.cqm_rmid = rmid;
895 static void intel_cqm_event_read(struct perf_event *event)
902 * Task events are handled by intel_cqm_event_count().
904 if (event->cpu == -1)
907 raw_spin_lock_irqsave(&cache_lock, flags);
908 rmid = event->hw.cqm_rmid;
910 if (!__rmid_valid(rmid))
913 val = __rmid_read(rmid);
916 * Ignore this reading on error states and do not update the value.
918 if (val & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL))
921 local64_set(&event->count, val);
923 raw_spin_unlock_irqrestore(&cache_lock, flags);
926 static void __intel_cqm_event_count(void *info)
928 struct rmid_read *rr = info;
931 val = __rmid_read(rr->rmid);
933 if (val & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL))
936 atomic64_add(val, &rr->value);
939 static inline bool cqm_group_leader(struct perf_event *event)
941 return !list_empty(&event->hw.cqm_groups_entry);
944 static u64 intel_cqm_event_count(struct perf_event *event)
947 struct rmid_read rr = {
948 .value = ATOMIC64_INIT(0),
952 * We only need to worry about task events. System-wide events
953 * are handled like usual, i.e. entirely with
954 * intel_cqm_event_read().
956 if (event->cpu != -1)
957 return __perf_event_count(event);
960 * Only the group leader gets to report values except in case of
961 * multiple events in the same group, we still need to read the
962 * other events.This stops us
963 * reporting duplicate values to userspace, and gives us a clear
964 * rule for which task gets to report the values.
966 * Note that it is impossible to attribute these values to
967 * specific packages - we forfeit that ability when we create
970 if (!cqm_group_leader(event) && !event->hw.is_group_event)
974 * Getting up-to-date values requires an SMP IPI which is not
975 * possible if we're being called in interrupt context. Return
976 * the cached values instead.
978 if (unlikely(in_interrupt()))
982 * Notice that we don't perform the reading of an RMID
983 * atomically, because we can't hold a spin lock across the
986 * Speculatively perform the read, since @event might be
987 * assigned a different (possibly invalid) RMID while we're
988 * busying performing the IPI calls. It's therefore necessary to
989 * check @event's RMID afterwards, and if it has changed,
990 * discard the result of the read.
992 rr.rmid = ACCESS_ONCE(event->hw.cqm_rmid);
994 if (!__rmid_valid(rr.rmid))
997 on_each_cpu_mask(&cqm_cpumask, __intel_cqm_event_count, &rr, 1);
999 raw_spin_lock_irqsave(&cache_lock, flags);
1000 if (event->hw.cqm_rmid == rr.rmid)
1001 local64_set(&event->count, atomic64_read(&rr.value));
1002 raw_spin_unlock_irqrestore(&cache_lock, flags);
1004 return __perf_event_count(event);
1007 static void intel_cqm_event_start(struct perf_event *event, int mode)
1009 struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
1010 u32 rmid = event->hw.cqm_rmid;
1012 if (!(event->hw.cqm_state & PERF_HES_STOPPED))
1015 event->hw.cqm_state &= ~PERF_HES_STOPPED;
1017 if (state->rmid_usecnt++) {
1018 if (!WARN_ON_ONCE(state->rmid != rmid))
1021 WARN_ON_ONCE(state->rmid);
1025 wrmsr(MSR_IA32_PQR_ASSOC, rmid, state->closid);
1028 static void intel_cqm_event_stop(struct perf_event *event, int mode)
1030 struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
1032 if (event->hw.cqm_state & PERF_HES_STOPPED)
1035 event->hw.cqm_state |= PERF_HES_STOPPED;
1037 intel_cqm_event_read(event);
1039 if (!--state->rmid_usecnt) {
1041 wrmsr(MSR_IA32_PQR_ASSOC, 0, state->closid);
1043 WARN_ON_ONCE(!state->rmid);
1047 static int intel_cqm_event_add(struct perf_event *event, int mode)
1049 unsigned long flags;
1052 raw_spin_lock_irqsave(&cache_lock, flags);
1054 event->hw.cqm_state = PERF_HES_STOPPED;
1055 rmid = event->hw.cqm_rmid;
1057 if (__rmid_valid(rmid) && (mode & PERF_EF_START))
1058 intel_cqm_event_start(event, mode);
1060 raw_spin_unlock_irqrestore(&cache_lock, flags);
1065 static void intel_cqm_event_destroy(struct perf_event *event)
1067 struct perf_event *group_other = NULL;
1069 mutex_lock(&cache_mutex);
1072 * If there's another event in this group...
1074 if (!list_empty(&event->hw.cqm_group_entry)) {
1075 group_other = list_first_entry(&event->hw.cqm_group_entry,
1077 hw.cqm_group_entry);
1078 list_del(&event->hw.cqm_group_entry);
1082 * And we're the group leader..
1084 if (cqm_group_leader(event)) {
1086 * If there was a group_other, make that leader, otherwise
1087 * destroy the group and return the RMID.
1090 list_replace(&event->hw.cqm_groups_entry,
1091 &group_other->hw.cqm_groups_entry);
1093 u32 rmid = event->hw.cqm_rmid;
1095 if (__rmid_valid(rmid))
1097 list_del(&event->hw.cqm_groups_entry);
1101 mutex_unlock(&cache_mutex);
1104 static int intel_cqm_event_init(struct perf_event *event)
1106 struct perf_event *group = NULL;
1107 bool rotate = false;
1109 if (event->attr.type != intel_cqm_pmu.type)
1112 if (event->attr.config & ~QOS_EVENT_MASK)
1115 /* unsupported modes and filters */
1116 if (event->attr.exclude_user ||
1117 event->attr.exclude_kernel ||
1118 event->attr.exclude_hv ||
1119 event->attr.exclude_idle ||
1120 event->attr.exclude_host ||
1121 event->attr.exclude_guest ||
1122 event->attr.sample_period) /* no sampling */
1125 INIT_LIST_HEAD(&event->hw.cqm_group_entry);
1126 INIT_LIST_HEAD(&event->hw.cqm_groups_entry);
1128 event->destroy = intel_cqm_event_destroy;
1130 mutex_lock(&cache_mutex);
1132 /* Will also set rmid */
1133 intel_cqm_setup_event(event, &group);
1136 list_add_tail(&event->hw.cqm_group_entry,
1137 &group->hw.cqm_group_entry);
1139 list_add_tail(&event->hw.cqm_groups_entry,
1143 * All RMIDs are either in use or have recently been
1144 * used. Kick the rotation worker to clean/free some.
1146 * We only do this for the group leader, rather than for
1147 * every event in a group to save on needless work.
1149 if (!__rmid_valid(event->hw.cqm_rmid))
1153 mutex_unlock(&cache_mutex);
1156 schedule_delayed_work(&intel_cqm_rmid_work, 0);
1161 EVENT_ATTR_STR(llc_occupancy, intel_cqm_llc, "event=0x01");
1162 EVENT_ATTR_STR(llc_occupancy.per-pkg, intel_cqm_llc_pkg, "1");
1163 EVENT_ATTR_STR(llc_occupancy.unit, intel_cqm_llc_unit, "Bytes");
1164 EVENT_ATTR_STR(llc_occupancy.scale, intel_cqm_llc_scale, NULL);
1165 EVENT_ATTR_STR(llc_occupancy.snapshot, intel_cqm_llc_snapshot, "1");
1167 static struct attribute *intel_cqm_events_attr[] = {
1168 EVENT_PTR(intel_cqm_llc),
1169 EVENT_PTR(intel_cqm_llc_pkg),
1170 EVENT_PTR(intel_cqm_llc_unit),
1171 EVENT_PTR(intel_cqm_llc_scale),
1172 EVENT_PTR(intel_cqm_llc_snapshot),
1176 static struct attribute_group intel_cqm_events_group = {
1178 .attrs = intel_cqm_events_attr,
1181 PMU_FORMAT_ATTR(event, "config:0-7");
1182 static struct attribute *intel_cqm_formats_attr[] = {
1183 &format_attr_event.attr,
1187 static struct attribute_group intel_cqm_format_group = {
1189 .attrs = intel_cqm_formats_attr,
1193 max_recycle_threshold_show(struct device *dev, struct device_attribute *attr,
1198 mutex_lock(&cache_mutex);
1199 rv = snprintf(page, PAGE_SIZE-1, "%u\n", __intel_cqm_max_threshold);
1200 mutex_unlock(&cache_mutex);
1206 max_recycle_threshold_store(struct device *dev,
1207 struct device_attribute *attr,
1208 const char *buf, size_t count)
1210 unsigned int bytes, cachelines;
1213 ret = kstrtouint(buf, 0, &bytes);
1217 mutex_lock(&cache_mutex);
1219 __intel_cqm_max_threshold = bytes;
1220 cachelines = bytes / cqm_l3_scale;
1223 * The new maximum takes effect immediately.
1225 if (__intel_cqm_threshold > cachelines)
1226 __intel_cqm_threshold = cachelines;
1228 mutex_unlock(&cache_mutex);
1233 static DEVICE_ATTR_RW(max_recycle_threshold);
1235 static struct attribute *intel_cqm_attrs[] = {
1236 &dev_attr_max_recycle_threshold.attr,
1240 static const struct attribute_group intel_cqm_group = {
1241 .attrs = intel_cqm_attrs,
1244 static const struct attribute_group *intel_cqm_attr_groups[] = {
1245 &intel_cqm_events_group,
1246 &intel_cqm_format_group,
1251 static struct pmu intel_cqm_pmu = {
1252 .hrtimer_interval_ms = RMID_DEFAULT_QUEUE_TIME,
1253 .attr_groups = intel_cqm_attr_groups,
1254 .task_ctx_nr = perf_sw_context,
1255 .event_init = intel_cqm_event_init,
1256 .add = intel_cqm_event_add,
1257 .del = intel_cqm_event_stop,
1258 .start = intel_cqm_event_start,
1259 .stop = intel_cqm_event_stop,
1260 .read = intel_cqm_event_read,
1261 .count = intel_cqm_event_count,
1264 static inline void cqm_pick_event_reader(int cpu)
1266 int phys_id = topology_physical_package_id(cpu);
1269 for_each_cpu(i, &cqm_cpumask) {
1270 if (phys_id == topology_physical_package_id(i))
1271 return; /* already got reader for this socket */
1274 cpumask_set_cpu(cpu, &cqm_cpumask);
1277 static void intel_cqm_cpu_starting(unsigned int cpu)
1279 struct intel_pqr_state *state = &per_cpu(pqr_state, cpu);
1280 struct cpuinfo_x86 *c = &cpu_data(cpu);
1284 state->rmid_usecnt = 0;
1286 WARN_ON(c->x86_cache_max_rmid != cqm_max_rmid);
1287 WARN_ON(c->x86_cache_occ_scale != cqm_l3_scale);
1290 static void intel_cqm_cpu_exit(unsigned int cpu)
1292 int phys_id = topology_physical_package_id(cpu);
1296 * Is @cpu a designated cqm reader?
1298 if (!cpumask_test_and_clear_cpu(cpu, &cqm_cpumask))
1301 for_each_online_cpu(i) {
1305 if (phys_id == topology_physical_package_id(i)) {
1306 cpumask_set_cpu(i, &cqm_cpumask);
1312 static int intel_cqm_cpu_notifier(struct notifier_block *nb,
1313 unsigned long action, void *hcpu)
1315 unsigned int cpu = (unsigned long)hcpu;
1317 switch (action & ~CPU_TASKS_FROZEN) {
1318 case CPU_DOWN_PREPARE:
1319 intel_cqm_cpu_exit(cpu);
1322 intel_cqm_cpu_starting(cpu);
1323 cqm_pick_event_reader(cpu);
1330 static const struct x86_cpu_id intel_cqm_match[] = {
1331 { .vendor = X86_VENDOR_INTEL, .feature = X86_FEATURE_CQM_OCCUP_LLC },
1335 static int __init intel_cqm_init(void)
1337 char *str = NULL, scale[20];
1340 if (!x86_match_cpu(intel_cqm_match))
1343 cqm_l3_scale = boot_cpu_data.x86_cache_occ_scale;
1346 * It's possible that not all resources support the same number
1347 * of RMIDs. Instead of making scheduling much more complicated
1348 * (where we have to match a task's RMID to a cpu that supports
1349 * that many RMIDs) just find the minimum RMIDs supported across
1352 * Also, check that the scales match on all cpus.
1354 cpu_notifier_register_begin();
1356 for_each_online_cpu(cpu) {
1357 struct cpuinfo_x86 *c = &cpu_data(cpu);
1359 if (c->x86_cache_max_rmid < cqm_max_rmid)
1360 cqm_max_rmid = c->x86_cache_max_rmid;
1362 if (c->x86_cache_occ_scale != cqm_l3_scale) {
1363 pr_err("Multiple LLC scale values, disabling\n");
1370 * A reasonable upper limit on the max threshold is the number
1371 * of lines tagged per RMID if all RMIDs have the same number of
1372 * lines tagged in the LLC.
1374 * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC.
1376 __intel_cqm_max_threshold =
1377 boot_cpu_data.x86_cache_size * 1024 / (cqm_max_rmid + 1);
1379 snprintf(scale, sizeof(scale), "%u", cqm_l3_scale);
1380 str = kstrdup(scale, GFP_KERNEL);
1386 event_attr_intel_cqm_llc_scale.event_str = str;
1388 ret = intel_cqm_setup_rmid_cache();
1392 for_each_online_cpu(i) {
1393 intel_cqm_cpu_starting(i);
1394 cqm_pick_event_reader(i);
1397 ret = perf_pmu_register(&intel_cqm_pmu, "intel_cqm", -1);
1399 pr_err("Intel CQM perf registration failed: %d\n", ret);
1403 pr_info("Intel CQM monitoring enabled\n");
1406 * Register the hot cpu notifier once we are sure cqm
1407 * is enabled to avoid notifier leak.
1409 __perf_cpu_notifier(intel_cqm_cpu_notifier);
1411 cpu_notifier_register_done();
1419 device_initcall(intel_cqm_init);