395 lines
11 KiB
Diff
395 lines
11 KiB
Diff
From fbebd5600c1b72bb707ada50d040ad719656a301 Mon Sep 17 00:00:00 2001
|
|
From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
|
|
Date: Mon, 28 Oct 2013 09:36:37 +0100
|
|
Subject: [PATCH] rtmutex: add a first shot of ww_mutex
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/3.14/patches-3.14.10-rt7.tar.xz
|
|
|
|
lockdep says:
|
|
| --------------------------------------------------------------------------
|
|
| | Wound/wait tests |
|
|
| ---------------------
|
|
| ww api failures: ok | ok | ok |
|
|
| ww contexts mixing: ok | ok |
|
|
| finishing ww context: ok | ok | ok | ok |
|
|
| locking mismatches: ok | ok | ok |
|
|
| EDEADLK handling: ok | ok | ok | ok | ok | ok | ok | ok | ok | ok |
|
|
| spinlock nest unlocked: ok |
|
|
| -----------------------------------------------------
|
|
| |block | try |context|
|
|
| -----------------------------------------------------
|
|
| context: ok | ok | ok |
|
|
| try: ok | ok | ok |
|
|
| block: ok | ok | ok |
|
|
| spinlock: ok | ok | ok |
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
|
|
---
|
|
kernel/locking/rtmutex.c | 235 ++++++++++++++++++++++++++++++++++++++++++++---
|
|
1 file changed, 222 insertions(+), 13 deletions(-)
|
|
|
|
Index: linux-stable/kernel/locking/rtmutex.c
|
|
===================================================================
|
|
--- linux-stable.orig/kernel/locking/rtmutex.c
|
|
+++ linux-stable/kernel/locking/rtmutex.c
|
|
@@ -22,6 +22,7 @@
|
|
#include <linux/sched/rt.h>
|
|
#include <linux/sched/deadline.h>
|
|
#include <linux/timer.h>
|
|
+#include <linux/ww_mutex.h>
|
|
|
|
#include "rtmutex_common.h"
|
|
|
|
@@ -1190,6 +1191,39 @@ EXPORT_SYMBOL(__rt_spin_lock_init);
|
|
|
|
#endif /* PREEMPT_RT_FULL */
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+static inline int __sched
|
|
+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
+{
|
|
+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
|
|
+ struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
|
|
+
|
|
+ if (!hold_ctx)
|
|
+ return 0;
|
|
+
|
|
+ if (unlikely(ctx == hold_ctx))
|
|
+ return -EALREADY;
|
|
+
|
|
+ if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
|
|
+ (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
|
|
+#ifdef CONFIG_DEBUG_MUTEXES
|
|
+ DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
|
|
+ ctx->contending_lock = ww;
|
|
+#endif
|
|
+ return -EDEADLK;
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+#else
|
|
+static inline int __sched
|
|
+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
+{
|
|
+ BUG();
|
|
+}
|
|
+
|
|
+#endif
|
|
+
|
|
/**
|
|
* __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
|
|
* @lock: the rt_mutex to take
|
|
@@ -1203,7 +1237,8 @@ EXPORT_SYMBOL(__rt_spin_lock_init);
|
|
static int __sched
|
|
__rt_mutex_slowlock(struct rt_mutex *lock, int state,
|
|
struct hrtimer_sleeper *timeout,
|
|
- struct rt_mutex_waiter *waiter)
|
|
+ struct rt_mutex_waiter *waiter,
|
|
+ struct ww_acquire_ctx *ww_ctx)
|
|
{
|
|
int ret = 0;
|
|
|
|
@@ -1226,6 +1261,12 @@ __rt_mutex_slowlock(struct rt_mutex *loc
|
|
break;
|
|
}
|
|
|
|
+ if (ww_ctx && ww_ctx->acquired > 0) {
|
|
+ ret = __mutex_lock_check_stamp(lock, ww_ctx);
|
|
+ if (ret)
|
|
+ break;
|
|
+ }
|
|
+
|
|
raw_spin_unlock(&lock->wait_lock);
|
|
|
|
debug_rt_mutex_print_deadlock(waiter);
|
|
@@ -1259,13 +1300,89 @@ static void rt_mutex_handle_deadlock(int
|
|
}
|
|
}
|
|
|
|
+static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
|
|
+ struct ww_acquire_ctx *ww_ctx)
|
|
+{
|
|
+#ifdef CONFIG_DEBUG_MUTEXES
|
|
+ /*
|
|
+ * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
|
|
+ * but released with a normal mutex_unlock in this call.
|
|
+ *
|
|
+ * This should never happen, always use ww_mutex_unlock.
|
|
+ */
|
|
+ DEBUG_LOCKS_WARN_ON(ww->ctx);
|
|
+
|
|
+ /*
|
|
+ * Not quite done after calling ww_acquire_done() ?
|
|
+ */
|
|
+ DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
|
|
+
|
|
+ if (ww_ctx->contending_lock) {
|
|
+ /*
|
|
+ * After -EDEADLK you tried to
|
|
+ * acquire a different ww_mutex? Bad!
|
|
+ */
|
|
+ DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
|
|
+
|
|
+ /*
|
|
+ * You called ww_mutex_lock after receiving -EDEADLK,
|
|
+ * but 'forgot' to unlock everything else first?
|
|
+ */
|
|
+ DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
|
|
+ ww_ctx->contending_lock = NULL;
|
|
+ }
|
|
+
|
|
+ /*
|
|
+ * Naughty, using a different class will lead to undefined behavior!
|
|
+ */
|
|
+ DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
|
|
+#endif
|
|
+ ww_ctx->acquired++;
|
|
+}
|
|
+
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+static void ww_mutex_account_lock(struct rt_mutex *lock,
|
|
+ struct ww_acquire_ctx *ww_ctx)
|
|
+{
|
|
+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
|
|
+ struct rt_mutex_waiter *waiter, *n;
|
|
+
|
|
+ /*
|
|
+ * This branch gets optimized out for the common case,
|
|
+ * and is only important for ww_mutex_lock.
|
|
+ */
|
|
+ ww_mutex_lock_acquired(ww, ww_ctx);
|
|
+ ww->ctx = ww_ctx;
|
|
+
|
|
+ /*
|
|
+ * Give any possible sleeping processes the chance to wake up,
|
|
+ * so they can recheck if they have to back off.
|
|
+ */
|
|
+ rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters,
|
|
+ tree_entry) {
|
|
+ /* XXX debug rt mutex waiter wakeup */
|
|
+
|
|
+ BUG_ON(waiter->lock != lock);
|
|
+ rt_mutex_wake_waiter(waiter);
|
|
+ }
|
|
+}
|
|
+
|
|
+#else
|
|
+
|
|
+static void ww_mutex_account_lock(struct rt_mutex *lock,
|
|
+ struct ww_acquire_ctx *ww_ctx)
|
|
+{
|
|
+ BUG();
|
|
+}
|
|
+#endif
|
|
+
|
|
/*
|
|
* Slow path lock function:
|
|
*/
|
|
static int __sched
|
|
rt_mutex_slowlock(struct rt_mutex *lock, int state,
|
|
struct hrtimer_sleeper *timeout,
|
|
- int detect_deadlock)
|
|
+ int detect_deadlock, struct ww_acquire_ctx *ww_ctx)
|
|
{
|
|
struct rt_mutex_waiter waiter;
|
|
int ret = 0;
|
|
@@ -1276,6 +1393,8 @@ rt_mutex_slowlock(struct rt_mutex *lock,
|
|
|
|
/* Try to acquire the lock again: */
|
|
if (try_to_take_rt_mutex(lock, current, NULL)) {
|
|
+ if (ww_ctx)
|
|
+ ww_mutex_account_lock(lock, ww_ctx);
|
|
raw_spin_unlock(&lock->wait_lock);
|
|
return 0;
|
|
}
|
|
@@ -1292,13 +1411,15 @@ rt_mutex_slowlock(struct rt_mutex *lock,
|
|
ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
|
|
|
|
if (likely(!ret))
|
|
- ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
|
|
+ ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, ww_ctx);
|
|
|
|
set_current_state(TASK_RUNNING);
|
|
|
|
if (unlikely(ret)) {
|
|
remove_waiter(lock, &waiter);
|
|
rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter);
|
|
+ } else if (ww_ctx) {
|
|
+ ww_mutex_account_lock(lock, ww_ctx);
|
|
}
|
|
|
|
/*
|
|
@@ -1414,30 +1535,33 @@ rt_mutex_slowunlock(struct rt_mutex *loc
|
|
*/
|
|
static inline int
|
|
rt_mutex_fastlock(struct rt_mutex *lock, int state,
|
|
- int detect_deadlock,
|
|
+ int detect_deadlock, struct ww_acquire_ctx *ww_ctx,
|
|
int (*slowfn)(struct rt_mutex *lock, int state,
|
|
struct hrtimer_sleeper *timeout,
|
|
- int detect_deadlock))
|
|
+ int detect_deadlock,
|
|
+ struct ww_acquire_ctx *ww_ctx))
|
|
{
|
|
if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
|
|
rt_mutex_deadlock_account_lock(lock, current);
|
|
return 0;
|
|
} else
|
|
- return slowfn(lock, state, NULL, detect_deadlock);
|
|
+ return slowfn(lock, state, NULL, detect_deadlock, ww_ctx);
|
|
}
|
|
|
|
static inline int
|
|
rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
|
|
struct hrtimer_sleeper *timeout, int detect_deadlock,
|
|
+ struct ww_acquire_ctx *ww_ctx,
|
|
int (*slowfn)(struct rt_mutex *lock, int state,
|
|
struct hrtimer_sleeper *timeout,
|
|
- int detect_deadlock))
|
|
+ int detect_deadlock,
|
|
+ struct ww_acquire_ctx *ww_ctx))
|
|
{
|
|
if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
|
|
rt_mutex_deadlock_account_lock(lock, current);
|
|
return 0;
|
|
} else
|
|
- return slowfn(lock, state, timeout, detect_deadlock);
|
|
+ return slowfn(lock, state, timeout, detect_deadlock, ww_ctx);
|
|
}
|
|
|
|
static inline int
|
|
@@ -1470,7 +1594,7 @@ void __sched rt_mutex_lock(struct rt_mut
|
|
{
|
|
might_sleep();
|
|
|
|
- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
|
|
+ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, rt_mutex_slowlock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(rt_mutex_lock);
|
|
|
|
@@ -1491,7 +1615,7 @@ int __sched rt_mutex_lock_interruptible(
|
|
might_sleep();
|
|
|
|
return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
|
|
- detect_deadlock, rt_mutex_slowlock);
|
|
+ detect_deadlock, NULL, rt_mutex_slowlock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
|
|
|
|
@@ -1512,7 +1636,7 @@ int __sched rt_mutex_lock_killable(struc
|
|
might_sleep();
|
|
|
|
return rt_mutex_fastlock(lock, TASK_KILLABLE,
|
|
- detect_deadlock, rt_mutex_slowlock);
|
|
+ detect_deadlock, NULL, rt_mutex_slowlock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
|
|
|
|
@@ -1538,7 +1662,7 @@ rt_mutex_timed_lock(struct rt_mutex *loc
|
|
might_sleep();
|
|
|
|
return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
|
|
- detect_deadlock, rt_mutex_slowlock);
|
|
+ detect_deadlock, NULL, rt_mutex_slowlock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
|
|
|
|
@@ -1764,7 +1888,7 @@ int rt_mutex_finish_proxy_lock(struct rt
|
|
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
|
|
- ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
|
|
+ ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL);
|
|
|
|
set_current_state(TASK_RUNNING);
|
|
|
|
@@ -1781,3 +1905,88 @@ int rt_mutex_finish_proxy_lock(struct rt
|
|
|
|
return ret;
|
|
}
|
|
+
|
|
+static inline int
|
|
+ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
+{
|
|
+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
|
|
+ unsigned tmp;
|
|
+
|
|
+ if (ctx->deadlock_inject_countdown-- == 0) {
|
|
+ tmp = ctx->deadlock_inject_interval;
|
|
+ if (tmp > UINT_MAX/4)
|
|
+ tmp = UINT_MAX;
|
|
+ else
|
|
+ tmp = tmp*2 + tmp + tmp/2;
|
|
+
|
|
+ ctx->deadlock_inject_interval = tmp;
|
|
+ ctx->deadlock_inject_countdown = tmp;
|
|
+ ctx->contending_lock = lock;
|
|
+
|
|
+ ww_mutex_unlock(lock);
|
|
+
|
|
+ return -EDEADLK;
|
|
+ }
|
|
+#endif
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+int __sched
|
|
+__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
|
|
+{
|
|
+ int ret;
|
|
+
|
|
+ might_sleep();
|
|
+
|
|
+ mutex_acquire(&lock->base.dep_map, 0, 0, _RET_IP_);
|
|
+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx);
|
|
+ if (ret)
|
|
+ mutex_release(&lock->base.dep_map, 1, _RET_IP_);
|
|
+ else if (!ret && ww_ctx->acquired > 1)
|
|
+ return ww_mutex_deadlock_injection(lock, ww_ctx);
|
|
+
|
|
+ return ret;
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
|
|
+
|
|
+int __sched
|
|
+__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
|
|
+{
|
|
+ int ret;
|
|
+
|
|
+ might_sleep();
|
|
+
|
|
+ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map,
|
|
+ _RET_IP_);
|
|
+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx);
|
|
+ if (ret)
|
|
+ mutex_release(&lock->base.dep_map, 1, _RET_IP_);
|
|
+ else if (!ret && ww_ctx->acquired > 1)
|
|
+ return ww_mutex_deadlock_injection(lock, ww_ctx);
|
|
+
|
|
+ return ret;
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(__ww_mutex_lock);
|
|
+
|
|
+void __sched ww_mutex_unlock(struct ww_mutex *lock)
|
|
+{
|
|
+ /*
|
|
+ * The unlocking fastpath is the 0->1 transition from 'locked'
|
|
+ * into 'unlocked' state:
|
|
+ */
|
|
+ if (lock->ctx) {
|
|
+#ifdef CONFIG_DEBUG_MUTEXES
|
|
+ DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
|
|
+#endif
|
|
+ if (lock->ctx->acquired > 0)
|
|
+ lock->ctx->acquired--;
|
|
+ lock->ctx = NULL;
|
|
+ }
|
|
+
|
|
+ mutex_release(&lock->base.dep_map, 1, _RET_IP_);
|
|
+ rt_mutex_unlock(&lock->base.lock);
|
|
+}
|
|
+EXPORT_SYMBOL(ww_mutex_unlock);
|
|
+#endif
|