These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / mm / list_lru.c

diff --git a/kernel/mm/list_lru.c b/kernel/mm/list_lru.c
index 909eca2..afc71ea 100644 (file)
--- a/kernel/mm/list_lru.c
+++ b/kernel/mm/list_lru.c
@@ -42,6 +42,10 @@ static void list_lru_unregister(struct list_lru *lru)
 #ifdef CONFIG_MEMCG_KMEM
 static inline bool list_lru_memcg_aware(struct list_lru *lru)
 {
+       /*
+        * This needs node 0 to be always present, even
+        * in the systems supporting sparse numa ids.
+        */
        return !!lru->node[0].memcg_lrus;
 }
 
@@ -59,6 +63,16 @@ list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
        return &nlru->lru;
 }
 
+static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
+{
+       struct page *page;
+
+       if (!memcg_kmem_enabled())
+               return NULL;
+       page = virt_to_head_page(ptr);
+       return page->mem_cgroup;
+}
+
 static inline struct list_lru_one *
 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
 {
@@ -99,8 +113,8 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item)
        struct list_lru_one *l;
 
        spin_lock(&nlru->lock);
-       l = list_lru_from_kmem(nlru, item);
        if (list_empty(item)) {
+               l = list_lru_from_kmem(nlru, item);
                list_add_tail(item, &l->list);
                l->nr_items++;
                spin_unlock(&nlru->lock);
@@ -118,8 +132,8 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
        struct list_lru_one *l;
 
        spin_lock(&nlru->lock);
-       l = list_lru_from_kmem(nlru, item);
        if (!list_empty(item)) {
+               l = list_lru_from_kmem(nlru, item);
                list_del_init(item);
                l->nr_items--;
                spin_unlock(&nlru->lock);
@@ -377,16 +391,20 @@ static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
 {
        int i;
 
-       for (i = 0; i < nr_node_ids; i++) {
-               if (!memcg_aware)
-                       lru->node[i].memcg_lrus = NULL;
-               else if (memcg_init_list_lru_node(&lru->node[i]))
+       if (!memcg_aware)
+               return 0;
+
+       for_each_node(i) {
+               if (memcg_init_list_lru_node(&lru->node[i]))
                        goto fail;
        }
        return 0;
 fail:
-       for (i = i - 1; i >= 0; i--)
+       for (i = i - 1; i >= 0; i--) {
+               if (!lru->node[i].memcg_lrus)
+                       continue;
                memcg_destroy_list_lru_node(&lru->node[i]);
+       }
        return -ENOMEM;
 }
 
@@ -397,7 +415,7 @@ static void memcg_destroy_list_lru(struct list_lru *lru)
        if (!list_lru_memcg_aware(lru))
                return;
 
-       for (i = 0; i < nr_node_ids; i++)
+       for_each_node(i)
                memcg_destroy_list_lru_node(&lru->node[i]);
 }
 
@@ -409,16 +427,20 @@ static int memcg_update_list_lru(struct list_lru *lru,
        if (!list_lru_memcg_aware(lru))
                return 0;
 
-       for (i = 0; i < nr_node_ids; i++) {
+       for_each_node(i) {
                if (memcg_update_list_lru_node(&lru->node[i],
                                               old_size, new_size))
                        goto fail;
        }
        return 0;
 fail:
-       for (i = i - 1; i >= 0; i--)
+       for (i = i - 1; i >= 0; i--) {
+               if (!lru->node[i].memcg_lrus)
+                       continue;
+
                memcg_cancel_update_list_lru_node(&lru->node[i],
                                                  old_size, new_size);
+       }
        return -ENOMEM;
 }
 
@@ -430,7 +452,7 @@ static void memcg_cancel_update_list_lru(struct list_lru *lru,
        if (!list_lru_memcg_aware(lru))
                return;
 
-       for (i = 0; i < nr_node_ids; i++)
+       for_each_node(i)
                memcg_cancel_update_list_lru_node(&lru->node[i],
                                                  old_size, new_size);
 }
@@ -485,7 +507,7 @@ static void memcg_drain_list_lru(struct list_lru *lru,
        if (!list_lru_memcg_aware(lru))
                return;
 
-       for (i = 0; i < nr_node_ids; i++)
+       for_each_node(i)
                memcg_drain_list_lru_node(&lru->node[i], src_idx, dst_idx);
 }
 
@@ -522,7 +544,7 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware,
        if (!lru->node)
                goto out;
 
-       for (i = 0; i < nr_node_ids; i++) {
+       for_each_node(i) {
                spin_lock_init(&lru->node[i].lock);
                if (key)
                        lockdep_set_class(&lru->node[i].lock, key);