2015-12-24 21:29:36 +00:00
|
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
Date: Wed, 31 Jul 2013 19:00:35 +0200
|
|
|
|
Subject: rcu: use simple waitqueues
|
2016-01-27 19:24:32 +00:00
|
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.4/patches-4.4-rt3.tar.xz
|
2015-12-24 21:29:36 +00:00
|
|
|
|
|
|
|
Convert RCU's wait-queues into simple waitqueues.
|
|
|
|
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
|
|
|
|
Merged Steven's
|
|
|
|
|
|
|
|
static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) {
|
|
|
|
- swait_wake(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
|
|
|
|
+ wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
|
|
|
|
}
|
|
|
|
|
|
|
|
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
|
|
---
|
|
|
|
kernel/rcu/tree.c | 16 ++++++++--------
|
|
|
|
kernel/rcu/tree.h | 9 +++++----
|
|
|
|
kernel/rcu/tree_plugin.h | 18 +++++++++---------
|
|
|
|
3 files changed, 22 insertions(+), 21 deletions(-)
|
|
|
|
|
|
|
|
--- a/kernel/rcu/tree.c
|
|
|
|
+++ b/kernel/rcu/tree.c
|
|
|
|
@@ -1633,7 +1633,7 @@ static void rcu_gp_kthread_wake(struct r
|
|
|
|
!READ_ONCE(rsp->gp_flags) ||
|
|
|
|
!rsp->gp_kthread)
|
|
|
|
return;
|
|
|
|
- wake_up(&rsp->gp_wq);
|
|
|
|
+ swait_wake(&rsp->gp_wq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
@@ -2098,7 +2098,7 @@ static int __noreturn rcu_gp_kthread(voi
|
|
|
|
READ_ONCE(rsp->gpnum),
|
|
|
|
TPS("reqwait"));
|
|
|
|
rsp->gp_state = RCU_GP_WAIT_GPS;
|
|
|
|
- wait_event_interruptible(rsp->gp_wq,
|
|
|
|
+ swait_event_interruptible(rsp->gp_wq,
|
|
|
|
READ_ONCE(rsp->gp_flags) &
|
|
|
|
RCU_GP_FLAG_INIT);
|
|
|
|
rsp->gp_state = RCU_GP_DONE_GPS;
|
|
|
|
@@ -2128,7 +2128,7 @@ static int __noreturn rcu_gp_kthread(voi
|
|
|
|
READ_ONCE(rsp->gpnum),
|
|
|
|
TPS("fqswait"));
|
|
|
|
rsp->gp_state = RCU_GP_WAIT_FQS;
|
|
|
|
- ret = wait_event_interruptible_timeout(rsp->gp_wq,
|
|
|
|
+ ret = swait_event_interruptible_timeout(rsp->gp_wq,
|
|
|
|
rcu_gp_fqs_check_wake(rsp, &gf), j);
|
|
|
|
rsp->gp_state = RCU_GP_DOING_FQS;
|
|
|
|
/* Locking provides needed memory barriers. */
|
|
|
|
@@ -3550,7 +3550,7 @@ static void __rcu_report_exp_rnp(struct
|
|
|
|
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
|
|
|
if (wake) {
|
|
|
|
smp_mb(); /* EGP done before wake_up(). */
|
|
|
|
- wake_up(&rsp->expedited_wq);
|
|
|
|
+ swait_wake(&rsp->expedited_wq);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
@@ -3807,7 +3807,7 @@ static void synchronize_sched_expedited_
|
|
|
|
jiffies_start = jiffies;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
- ret = wait_event_interruptible_timeout(
|
|
|
|
+ ret = swait_event_interruptible_timeout(
|
|
|
|
rsp->expedited_wq,
|
|
|
|
sync_rcu_preempt_exp_done(rnp_root),
|
|
|
|
jiffies_stall);
|
|
|
|
@@ -3815,7 +3815,7 @@ static void synchronize_sched_expedited_
|
|
|
|
return;
|
|
|
|
if (ret < 0) {
|
|
|
|
/* Hit a signal, disable CPU stall warnings. */
|
|
|
|
- wait_event(rsp->expedited_wq,
|
|
|
|
+ swait_event(rsp->expedited_wq,
|
|
|
|
sync_rcu_preempt_exp_done(rnp_root));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
@@ -4483,8 +4483,8 @@ static void __init rcu_init_one(struct r
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
- init_waitqueue_head(&rsp->gp_wq);
|
|
|
|
- init_waitqueue_head(&rsp->expedited_wq);
|
|
|
|
+ init_swait_head(&rsp->gp_wq);
|
|
|
|
+ init_swait_head(&rsp->expedited_wq);
|
|
|
|
rnp = rsp->level[rcu_num_lvls - 1];
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
while (i > rnp->grphi)
|
|
|
|
--- a/kernel/rcu/tree.h
|
|
|
|
+++ b/kernel/rcu/tree.h
|
|
|
|
@@ -28,6 +28,7 @@
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <linux/seqlock.h>
|
|
|
|
#include <linux/stop_machine.h>
|
|
|
|
+#include <linux/wait-simple.h>
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and
|
|
|
|
@@ -241,7 +242,7 @@ struct rcu_node {
|
|
|
|
/* Refused to boost: not sure why, though. */
|
|
|
|
/* This can happen due to race conditions. */
|
|
|
|
#ifdef CONFIG_RCU_NOCB_CPU
|
|
|
|
- wait_queue_head_t nocb_gp_wq[2];
|
|
|
|
+ struct swait_head nocb_gp_wq[2];
|
|
|
|
/* Place for rcu_nocb_kthread() to wait GP. */
|
|
|
|
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
|
|
|
|
int need_future_gp[2];
|
|
|
|
@@ -393,7 +394,7 @@ struct rcu_data {
|
|
|
|
atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */
|
|
|
|
struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */
|
|
|
|
struct rcu_head **nocb_follower_tail;
|
|
|
|
- wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */
|
|
|
|
+ struct swait_head nocb_wq; /* For nocb kthreads to sleep on. */
|
|
|
|
struct task_struct *nocb_kthread;
|
|
|
|
int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */
|
|
|
|
|
|
|
|
@@ -472,7 +473,7 @@ struct rcu_state {
|
|
|
|
unsigned long gpnum; /* Current gp number. */
|
|
|
|
unsigned long completed; /* # of last completed gp. */
|
|
|
|
struct task_struct *gp_kthread; /* Task for grace periods. */
|
|
|
|
- wait_queue_head_t gp_wq; /* Where GP task waits. */
|
|
|
|
+ struct swait_head gp_wq; /* Where GP task waits. */
|
|
|
|
short gp_flags; /* Commands for GP task. */
|
|
|
|
short gp_state; /* GP kthread sleep state. */
|
|
|
|
|
|
|
|
@@ -504,7 +505,7 @@ struct rcu_state {
|
|
|
|
atomic_long_t expedited_workdone3; /* # done by others #3. */
|
|
|
|
atomic_long_t expedited_normal; /* # fallbacks to normal. */
|
|
|
|
atomic_t expedited_need_qs; /* # CPUs left to check in. */
|
|
|
|
- wait_queue_head_t expedited_wq; /* Wait for check-ins. */
|
|
|
|
+ struct swait_head expedited_wq; /* Wait for check-ins. */
|
|
|
|
int ncpus_snap; /* # CPUs seen last time. */
|
|
|
|
|
|
|
|
unsigned long jiffies_force_qs; /* Time at which to invoke */
|
|
|
|
--- a/kernel/rcu/tree_plugin.h
|
|
|
|
+++ b/kernel/rcu/tree_plugin.h
|
|
|
|
@@ -1830,7 +1830,7 @@ early_param("rcu_nocb_poll", parse_rcu_n
|
|
|
|
*/
|
|
|
|
static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
|
|
|
|
{
|
|
|
|
- wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
|
|
|
|
+ swait_wake_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
@@ -1848,8 +1848,8 @@ static void rcu_nocb_gp_set(struct rcu_n
|
|
|
|
|
|
|
|
static void rcu_init_one_nocb(struct rcu_node *rnp)
|
|
|
|
{
|
|
|
|
- init_waitqueue_head(&rnp->nocb_gp_wq[0]);
|
|
|
|
- init_waitqueue_head(&rnp->nocb_gp_wq[1]);
|
|
|
|
+ init_swait_head(&rnp->nocb_gp_wq[0]);
|
|
|
|
+ init_swait_head(&rnp->nocb_gp_wq[1]);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef CONFIG_RCU_NOCB_CPU_ALL
|
|
|
|
@@ -1874,7 +1874,7 @@ static void wake_nocb_leader(struct rcu_
|
|
|
|
if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
|
|
|
|
/* Prior smp_mb__after_atomic() orders against prior enqueue. */
|
|
|
|
WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
|
|
|
|
- wake_up(&rdp_leader->nocb_wq);
|
|
|
|
+ swait_wake(&rdp_leader->nocb_wq);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@@ -2087,7 +2087,7 @@ static void rcu_nocb_wait_gp(struct rcu_
|
|
|
|
*/
|
|
|
|
trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
|
|
|
|
for (;;) {
|
|
|
|
- wait_event_interruptible(
|
|
|
|
+ swait_event_interruptible(
|
|
|
|
rnp->nocb_gp_wq[c & 0x1],
|
|
|
|
(d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
|
|
|
|
if (likely(d))
|
|
|
|
@@ -2115,7 +2115,7 @@ static void nocb_leader_wait(struct rcu_
|
|
|
|
/* Wait for callbacks to appear. */
|
|
|
|
if (!rcu_nocb_poll) {
|
|
|
|
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
|
|
|
|
- wait_event_interruptible(my_rdp->nocb_wq,
|
|
|
|
+ swait_event_interruptible(my_rdp->nocb_wq,
|
|
|
|
!READ_ONCE(my_rdp->nocb_leader_sleep));
|
|
|
|
/* Memory barrier handled by smp_mb() calls below and repoll. */
|
|
|
|
} else if (firsttime) {
|
|
|
|
@@ -2190,7 +2190,7 @@ static void nocb_leader_wait(struct rcu_
|
|
|
|
* List was empty, wake up the follower.
|
|
|
|
* Memory barriers supplied by atomic_long_add().
|
|
|
|
*/
|
|
|
|
- wake_up(&rdp->nocb_wq);
|
|
|
|
+ swait_wake(&rdp->nocb_wq);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@@ -2211,7 +2211,7 @@ static void nocb_follower_wait(struct rc
|
|
|
|
if (!rcu_nocb_poll) {
|
|
|
|
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
|
|
|
|
"FollowerSleep");
|
|
|
|
- wait_event_interruptible(rdp->nocb_wq,
|
|
|
|
+ swait_event_interruptible(rdp->nocb_wq,
|
|
|
|
READ_ONCE(rdp->nocb_follower_head));
|
|
|
|
} else if (firsttime) {
|
|
|
|
/* Don't drown trace log with "Poll"! */
|
|
|
|
@@ -2370,7 +2370,7 @@ void __init rcu_init_nohz(void)
|
|
|
|
static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
|
|
|
|
{
|
|
|
|
rdp->nocb_tail = &rdp->nocb_head;
|
|
|
|
- init_waitqueue_head(&rdp->nocb_wq);
|
|
|
|
+ init_swait_head(&rdp->nocb_wq);
|
|
|
|
rdp->nocb_follower_tail = &rdp->nocb_follower_head;
|
|
|
|
}
|
|
|
|
|