146 lines
5.0 KiB
Diff
146 lines
5.0 KiB
Diff
From: Anna-Maria Gleixner <anna-maria@linutronix.de>
|
|
Date: Wed, 20 Dec 2017 17:13:14 +0100
|
|
Subject: [PATCH 25/29] hrtimer: Use irqsave/irqrestore around __run_hrtimer()
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.14/older/patches-4.14.15-rt13.tar.xz
|
|
|
|
__run_hrtimer() is called with the hrtimer_cpu_base.lock held and
|
|
interrupts disabled. Before invoking the timer callback the base lock is
|
|
dropped, but interrupts stay disabled.
|
|
|
|
The upcoming support for softirq based hrtimers requires that interrupts
|
|
are enabled before the timer callback is invoked.
|
|
|
|
To avoid code duplication, take hrtimer_cpu_base.lock with
|
|
raw_spin_lock_irqsave(flags) at the call site and hand in the flags as
|
|
argument. So raw_spin_unlock_irqrestore() before the callback invocation
|
|
will either keep interrupts disabled in interrupt context or restore to
|
|
interrupt enabled state when called from softirq context.
|
|
|
|
Suggested-by: Peter Zijlstra <peterz@infradead.org>
|
|
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
---
|
|
kernel/time/hrtimer.c | 31 ++++++++++++++++++-------------
|
|
1 file changed, 18 insertions(+), 13 deletions(-)
|
|
|
|
--- a/kernel/time/hrtimer.c
|
|
+++ b/kernel/time/hrtimer.c
|
|
@@ -1163,7 +1163,8 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
|
|
|
|
static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
|
|
struct hrtimer_clock_base *base,
|
|
- struct hrtimer *timer, ktime_t *now)
|
|
+ struct hrtimer *timer, ktime_t *now,
|
|
+ unsigned long flags)
|
|
{
|
|
enum hrtimer_restart (*fn)(struct hrtimer *);
|
|
int restart;
|
|
@@ -1198,11 +1199,11 @@ static void __run_hrtimer(struct hrtimer
|
|
* protected against migration to a different CPU even if the lock
|
|
* is dropped.
|
|
*/
|
|
- raw_spin_unlock(&cpu_base->lock);
|
|
+ raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
trace_hrtimer_expire_entry(timer, now);
|
|
restart = fn(timer);
|
|
trace_hrtimer_expire_exit(timer);
|
|
- raw_spin_lock(&cpu_base->lock);
|
|
+ raw_spin_lock_irq(&cpu_base->lock);
|
|
|
|
/*
|
|
* Note: We clear the running state after enqueue_hrtimer and
|
|
@@ -1230,7 +1231,8 @@ static void __run_hrtimer(struct hrtimer
|
|
base->running = NULL;
|
|
}
|
|
|
|
-static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
|
|
+static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
|
|
+ unsigned long flags)
|
|
{
|
|
struct hrtimer_clock_base *base;
|
|
unsigned int active = cpu_base->active_bases;
|
|
@@ -1261,7 +1263,7 @@ static void __hrtimer_run_queues(struct
|
|
if (basenow < hrtimer_get_softexpires_tv64(timer))
|
|
break;
|
|
|
|
- __run_hrtimer(cpu_base, base, timer, &basenow);
|
|
+ __run_hrtimer(cpu_base, base, timer, &basenow, flags);
|
|
}
|
|
}
|
|
}
|
|
@@ -1276,13 +1278,14 @@ void hrtimer_interrupt(struct clock_even
|
|
{
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
ktime_t expires_next, now, entry_time, delta;
|
|
+ unsigned long flags;
|
|
int retries = 0;
|
|
|
|
BUG_ON(!cpu_base->hres_active);
|
|
cpu_base->nr_events++;
|
|
dev->next_event = KTIME_MAX;
|
|
|
|
- raw_spin_lock(&cpu_base->lock);
|
|
+ raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
|
entry_time = now = hrtimer_update_base(cpu_base);
|
|
retry:
|
|
cpu_base->in_hrtirq = 1;
|
|
@@ -1295,7 +1298,7 @@ void hrtimer_interrupt(struct clock_even
|
|
*/
|
|
cpu_base->expires_next = KTIME_MAX;
|
|
|
|
- __hrtimer_run_queues(cpu_base, now);
|
|
+ __hrtimer_run_queues(cpu_base, now, flags);
|
|
|
|
/* Reevaluate the clock bases for the next expiry */
|
|
expires_next = __hrtimer_get_next_event(cpu_base);
|
|
@@ -1305,7 +1308,7 @@ void hrtimer_interrupt(struct clock_even
|
|
*/
|
|
cpu_base->expires_next = expires_next;
|
|
cpu_base->in_hrtirq = 0;
|
|
- raw_spin_unlock(&cpu_base->lock);
|
|
+ raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
/* Reprogramming necessary ? */
|
|
if (!tick_program_event(expires_next, 0)) {
|
|
@@ -1326,7 +1329,7 @@ void hrtimer_interrupt(struct clock_even
|
|
* Acquire base lock for updating the offsets and retrieving
|
|
* the current time.
|
|
*/
|
|
- raw_spin_lock(&cpu_base->lock);
|
|
+ raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
|
now = hrtimer_update_base(cpu_base);
|
|
cpu_base->nr_retries++;
|
|
if (++retries < 3)
|
|
@@ -1339,7 +1342,8 @@ void hrtimer_interrupt(struct clock_even
|
|
*/
|
|
cpu_base->nr_hangs++;
|
|
cpu_base->hang_detected = 1;
|
|
- raw_spin_unlock(&cpu_base->lock);
|
|
+ raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
+
|
|
delta = ktime_sub(now, entry_time);
|
|
if ((unsigned int)delta > cpu_base->max_hang_time)
|
|
cpu_base->max_hang_time = (unsigned int) delta;
|
|
@@ -1381,6 +1385,7 @@ static inline void __hrtimer_peek_ahead_
|
|
void hrtimer_run_queues(void)
|
|
{
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
+ unsigned long flags;
|
|
ktime_t now;
|
|
|
|
if (__hrtimer_hres_active(cpu_base))
|
|
@@ -1398,10 +1403,10 @@ void hrtimer_run_queues(void)
|
|
return;
|
|
}
|
|
|
|
- raw_spin_lock(&cpu_base->lock);
|
|
+ raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
|
now = hrtimer_update_base(cpu_base);
|
|
- __hrtimer_run_queues(cpu_base, now);
|
|
- raw_spin_unlock(&cpu_base->lock);
|
|
+ __hrtimer_run_queues(cpu_base, now, flags);
|
|
+ raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
}
|
|
|
|
/*
|