From: Sebastian Andrzej Siewior Date: Thu, 29 Jan 2015 17:19:44 +0100 Subject: mm/workingset: Do not protect workingset_shadow_nodes with irq off Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.14/older/patches-4.14.8-rt9.tar.xz workingset_shadow_nodes is protected by local_irq_disable(). Some users use spin_lock_irq(). Replace the irq/on with a local_lock(). Rename workingset_shadow_nodes so I catch users of it which will be introduced later. Signed-off-by: Sebastian Andrzej Siewior --- include/linux/swap.h | 4 +++- mm/filemap.c | 9 +++++++-- mm/truncate.c | 4 +++- mm/workingset.c | 31 ++++++++++++++++--------------- 4 files changed, 29 insertions(+), 19 deletions(-) --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -12,6 +12,7 @@ #include #include #include +#include #include struct notifier_block; @@ -297,7 +298,8 @@ struct vma_swap_readahead { void *workingset_eviction(struct address_space *mapping, struct page *page); bool workingset_refault(void *shadow); void workingset_activation(struct page *page); -void workingset_update_node(struct radix_tree_node *node, void *private); +void __workingset_update_node(struct radix_tree_node *node, void *private); +DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock); /* linux/mm/page_alloc.c */ extern unsigned long totalram_pages; --- a/mm/filemap.c +++ b/mm/filemap.c @@ -110,6 +110,7 @@ * ->i_mmap_rwsem * ->tasklist_lock (memory_failure, collect_procs_ao) */ +DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock); static int page_cache_tree_insert(struct address_space *mapping, struct page *page, void **shadowp) @@ -133,8 +134,10 @@ static int page_cache_tree_insert(struct if (shadowp) *shadowp = p; } + local_lock(shadow_nodes_lock); __radix_tree_replace(&mapping->page_tree, node, slot, page, - workingset_update_node, mapping); + __workingset_update_node, mapping); + local_unlock(shadow_nodes_lock); mapping->nrpages++; return 0; } @@ -151,6 +154,7 @@ static void page_cache_tree_delete(struc VM_BUG_ON_PAGE(PageTail(page), page); VM_BUG_ON_PAGE(nr != 1 && shadow, page); + local_lock(shadow_nodes_lock); for (i = 0; i < nr; i++) { struct radix_tree_node *node; void **slot; @@ -162,8 +166,9 @@ static void page_cache_tree_delete(struc radix_tree_clear_tags(&mapping->page_tree, node, slot); __radix_tree_replace(&mapping->page_tree, node, slot, shadow, - workingset_update_node, mapping); + __workingset_update_node, mapping); } + local_unlock(shadow_nodes_lock); if (shadow) { mapping->nrexceptional += nr; --- a/mm/truncate.c +++ b/mm/truncate.c @@ -41,8 +41,10 @@ static void clear_shadow_entry(struct ad goto unlock; if (*slot != entry) goto unlock; + local_lock(shadow_nodes_lock); __radix_tree_replace(&mapping->page_tree, node, slot, NULL, - workingset_update_node, mapping); + __workingset_update_node, mapping); + local_unlock(shadow_nodes_lock); mapping->nrexceptional--; unlock: spin_unlock_irq(&mapping->tree_lock); --- a/mm/workingset.c +++ b/mm/workingset.c @@ -338,9 +338,10 @@ void workingset_activation(struct page * * point where they would still be useful. */ -static struct list_lru shadow_nodes; +static struct list_lru __shadow_nodes; +DEFINE_LOCAL_IRQ_LOCK(shadow_nodes_lock); -void workingset_update_node(struct radix_tree_node *node, void *private) +void __workingset_update_node(struct radix_tree_node *node, void *private) { struct address_space *mapping = private; @@ -358,10 +359,10 @@ void workingset_update_node(struct radix */ if (node->count && node->count == node->exceptional) { if (list_empty(&node->private_list)) - list_lru_add(&shadow_nodes, &node->private_list); + list_lru_add(&__shadow_nodes, &node->private_list); } else { if (!list_empty(&node->private_list)) - list_lru_del(&shadow_nodes, &node->private_list); + list_lru_del(&__shadow_nodes, &node->private_list); } } @@ -373,9 +374,9 @@ static unsigned long count_shadow_nodes( unsigned long cache; /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ - local_irq_disable(); - nodes = list_lru_shrink_count(&shadow_nodes, sc); - local_irq_enable(); + local_lock_irq(shadow_nodes_lock); + nodes = list_lru_shrink_count(&__shadow_nodes, sc); + local_unlock_irq(shadow_nodes_lock); /* * Approximate a reasonable limit for the radix tree nodes @@ -475,15 +476,15 @@ static enum lru_status shadow_lru_isolat goto out_invalid; inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); __radix_tree_delete_node(&mapping->page_tree, node, - workingset_update_node, mapping); + __workingset_update_node, mapping); out_invalid: spin_unlock(&mapping->tree_lock); ret = LRU_REMOVED_RETRY; out: - local_irq_enable(); + local_unlock_irq(shadow_nodes_lock); cond_resched(); - local_irq_disable(); + local_lock_irq(shadow_nodes_lock); spin_lock(lru_lock); return ret; } @@ -494,9 +495,9 @@ static unsigned long scan_shadow_nodes(s unsigned long ret; /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ - local_irq_disable(); - ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); - local_irq_enable(); + local_lock_irq(shadow_nodes_lock); + ret = list_lru_shrink_walk(&__shadow_nodes, sc, shadow_lru_isolate, NULL); + local_unlock_irq(shadow_nodes_lock); return ret; } @@ -534,7 +535,7 @@ static int __init workingset_init(void) pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n", timestamp_bits, max_order, bucket_order); - ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key); + ret = __list_lru_init(&__shadow_nodes, true, &shadow_nodes_key); if (ret) goto err; ret = register_shrinker(&workingset_shadow_shrinker); @@ -542,7 +543,7 @@ static int __init workingset_init(void) goto err_list_lru; return 0; err_list_lru: - list_lru_destroy(&shadow_nodes); + list_lru_destroy(&__shadow_nodes); err: return ret; }