185 lines
6.0 KiB
Diff
185 lines
6.0 KiB
Diff
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
Date: Thu, 29 Jan 2015 17:19:44 +0100
|
|
Subject: mm/workingset: Do not protect workingset_shadow_nodes with irq off
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.14/older/patches-4.14.8-rt9.tar.xz
|
|
|
|
workingset_shadow_nodes is protected by local_irq_disable(). Some users
|
|
use spin_lock_irq().
|
|
Replace the irq/on with a local_lock(). Rename workingset_shadow_nodes
|
|
so I catch users of it which will be introduced later.
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
---
|
|
include/linux/swap.h | 4 +++-
|
|
mm/filemap.c | 9 +++++++--
|
|
mm/truncate.c | 4 +++-
|
|
mm/workingset.c | 31 ++++++++++++++++---------------
|
|
4 files changed, 29 insertions(+), 19 deletions(-)
|
|
|
|
--- a/include/linux/swap.h
|
|
+++ b/include/linux/swap.h
|
|
@@ -12,6 +12,7 @@
|
|
#include <linux/fs.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/page-flags.h>
|
|
+#include <linux/locallock.h>
|
|
#include <asm/page.h>
|
|
|
|
struct notifier_block;
|
|
@@ -297,7 +298,8 @@ struct vma_swap_readahead {
|
|
void *workingset_eviction(struct address_space *mapping, struct page *page);
|
|
bool workingset_refault(void *shadow);
|
|
void workingset_activation(struct page *page);
|
|
-void workingset_update_node(struct radix_tree_node *node, void *private);
|
|
+void __workingset_update_node(struct radix_tree_node *node, void *private);
|
|
+DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock);
|
|
|
|
/* linux/mm/page_alloc.c */
|
|
extern unsigned long totalram_pages;
|
|
--- a/mm/filemap.c
|
|
+++ b/mm/filemap.c
|
|
@@ -110,6 +110,7 @@
|
|
* ->i_mmap_rwsem
|
|
* ->tasklist_lock (memory_failure, collect_procs_ao)
|
|
*/
|
|
+DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock);
|
|
|
|
static int page_cache_tree_insert(struct address_space *mapping,
|
|
struct page *page, void **shadowp)
|
|
@@ -133,8 +134,10 @@ static int page_cache_tree_insert(struct
|
|
if (shadowp)
|
|
*shadowp = p;
|
|
}
|
|
+ local_lock(shadow_nodes_lock);
|
|
__radix_tree_replace(&mapping->page_tree, node, slot, page,
|
|
- workingset_update_node, mapping);
|
|
+ __workingset_update_node, mapping);
|
|
+ local_unlock(shadow_nodes_lock);
|
|
mapping->nrpages++;
|
|
return 0;
|
|
}
|
|
@@ -151,6 +154,7 @@ static void page_cache_tree_delete(struc
|
|
VM_BUG_ON_PAGE(PageTail(page), page);
|
|
VM_BUG_ON_PAGE(nr != 1 && shadow, page);
|
|
|
|
+ local_lock(shadow_nodes_lock);
|
|
for (i = 0; i < nr; i++) {
|
|
struct radix_tree_node *node;
|
|
void **slot;
|
|
@@ -162,8 +166,9 @@ static void page_cache_tree_delete(struc
|
|
|
|
radix_tree_clear_tags(&mapping->page_tree, node, slot);
|
|
__radix_tree_replace(&mapping->page_tree, node, slot, shadow,
|
|
- workingset_update_node, mapping);
|
|
+ __workingset_update_node, mapping);
|
|
}
|
|
+ local_unlock(shadow_nodes_lock);
|
|
|
|
if (shadow) {
|
|
mapping->nrexceptional += nr;
|
|
--- a/mm/truncate.c
|
|
+++ b/mm/truncate.c
|
|
@@ -41,8 +41,10 @@ static void clear_shadow_entry(struct ad
|
|
goto unlock;
|
|
if (*slot != entry)
|
|
goto unlock;
|
|
+ local_lock(shadow_nodes_lock);
|
|
__radix_tree_replace(&mapping->page_tree, node, slot, NULL,
|
|
- workingset_update_node, mapping);
|
|
+ __workingset_update_node, mapping);
|
|
+ local_unlock(shadow_nodes_lock);
|
|
mapping->nrexceptional--;
|
|
unlock:
|
|
spin_unlock_irq(&mapping->tree_lock);
|
|
--- a/mm/workingset.c
|
|
+++ b/mm/workingset.c
|
|
@@ -338,9 +338,10 @@ void workingset_activation(struct page *
|
|
* point where they would still be useful.
|
|
*/
|
|
|
|
-static struct list_lru shadow_nodes;
|
|
+static struct list_lru __shadow_nodes;
|
|
+DEFINE_LOCAL_IRQ_LOCK(shadow_nodes_lock);
|
|
|
|
-void workingset_update_node(struct radix_tree_node *node, void *private)
|
|
+void __workingset_update_node(struct radix_tree_node *node, void *private)
|
|
{
|
|
struct address_space *mapping = private;
|
|
|
|
@@ -358,10 +359,10 @@ void workingset_update_node(struct radix
|
|
*/
|
|
if (node->count && node->count == node->exceptional) {
|
|
if (list_empty(&node->private_list))
|
|
- list_lru_add(&shadow_nodes, &node->private_list);
|
|
+ list_lru_add(&__shadow_nodes, &node->private_list);
|
|
} else {
|
|
if (!list_empty(&node->private_list))
|
|
- list_lru_del(&shadow_nodes, &node->private_list);
|
|
+ list_lru_del(&__shadow_nodes, &node->private_list);
|
|
}
|
|
}
|
|
|
|
@@ -373,9 +374,9 @@ static unsigned long count_shadow_nodes(
|
|
unsigned long cache;
|
|
|
|
/* list_lru lock nests inside IRQ-safe mapping->tree_lock */
|
|
- local_irq_disable();
|
|
- nodes = list_lru_shrink_count(&shadow_nodes, sc);
|
|
- local_irq_enable();
|
|
+ local_lock_irq(shadow_nodes_lock);
|
|
+ nodes = list_lru_shrink_count(&__shadow_nodes, sc);
|
|
+ local_unlock_irq(shadow_nodes_lock);
|
|
|
|
/*
|
|
* Approximate a reasonable limit for the radix tree nodes
|
|
@@ -475,15 +476,15 @@ static enum lru_status shadow_lru_isolat
|
|
goto out_invalid;
|
|
inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM);
|
|
__radix_tree_delete_node(&mapping->page_tree, node,
|
|
- workingset_update_node, mapping);
|
|
+ __workingset_update_node, mapping);
|
|
|
|
out_invalid:
|
|
spin_unlock(&mapping->tree_lock);
|
|
ret = LRU_REMOVED_RETRY;
|
|
out:
|
|
- local_irq_enable();
|
|
+ local_unlock_irq(shadow_nodes_lock);
|
|
cond_resched();
|
|
- local_irq_disable();
|
|
+ local_lock_irq(shadow_nodes_lock);
|
|
spin_lock(lru_lock);
|
|
return ret;
|
|
}
|
|
@@ -494,9 +495,9 @@ static unsigned long scan_shadow_nodes(s
|
|
unsigned long ret;
|
|
|
|
/* list_lru lock nests inside IRQ-safe mapping->tree_lock */
|
|
- local_irq_disable();
|
|
- ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL);
|
|
- local_irq_enable();
|
|
+ local_lock_irq(shadow_nodes_lock);
|
|
+ ret = list_lru_shrink_walk(&__shadow_nodes, sc, shadow_lru_isolate, NULL);
|
|
+ local_unlock_irq(shadow_nodes_lock);
|
|
return ret;
|
|
}
|
|
|
|
@@ -534,7 +535,7 @@ static int __init workingset_init(void)
|
|
pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
|
|
timestamp_bits, max_order, bucket_order);
|
|
|
|
- ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key);
|
|
+ ret = __list_lru_init(&__shadow_nodes, true, &shadow_nodes_key);
|
|
if (ret)
|
|
goto err;
|
|
ret = register_shrinker(&workingset_shadow_shrinker);
|
|
@@ -542,7 +543,7 @@ static int __init workingset_init(void)
|
|
goto err_list_lru;
|
|
return 0;
|
|
err_list_lru:
|
|
- list_lru_destroy(&shadow_nodes);
|
|
+ list_lru_destroy(&__shadow_nodes);
|
|
err:
|
|
return ret;
|
|
}
|