273 lines
8.3 KiB
Diff
273 lines
8.3 KiB
Diff
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
Date: Fri, 5 Feb 2016 18:26:11 +0100
|
|
Subject: rtmutex: push down migrate_disable() into rt_spin_lock()
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.6/older/patches-4.6.2-rt5.tar.xz
|
|
|
|
No point in having the migrate disable/enable invocations in all the
|
|
macro/inlines. That's just more code for no win as we do a function
|
|
call anyway. Move it to the core code and save quite some text size.
|
|
|
|
text data bss dec filename
|
|
11034127 3676912 14901248 29612287 vmlinux.before
|
|
10990437 3676848 14901248 29568533 vmlinux.after
|
|
|
|
~-40KiB
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
---
|
|
include/linux/locallock.h | 6 +++---
|
|
include/linux/spinlock_rt.h | 25 +++++++------------------
|
|
kernel/cpu.c | 4 ++--
|
|
kernel/locking/lglock.c | 2 +-
|
|
kernel/locking/rt.c | 2 --
|
|
kernel/locking/rtmutex.c | 44 +++++++++++++++++++++++++++++++++++++++++---
|
|
6 files changed, 54 insertions(+), 29 deletions(-)
|
|
|
|
--- a/include/linux/locallock.h
|
|
+++ b/include/linux/locallock.h
|
|
@@ -43,9 +43,9 @@ struct local_irq_lock {
|
|
* for CONFIG_PREEMPT_BASE map to the normal spin_* calls.
|
|
*/
|
|
#ifdef CONFIG_PREEMPT_RT_FULL
|
|
-# define spin_lock_local(lock) rt_spin_lock(lock)
|
|
-# define spin_trylock_local(lock) rt_spin_trylock(lock)
|
|
-# define spin_unlock_local(lock) rt_spin_unlock(lock)
|
|
+# define spin_lock_local(lock) rt_spin_lock__no_mg(lock)
|
|
+# define spin_trylock_local(lock) rt_spin_trylock__no_mg(lock)
|
|
+# define spin_unlock_local(lock) rt_spin_unlock__no_mg(lock)
|
|
#else
|
|
# define spin_lock_local(lock) spin_lock(lock)
|
|
# define spin_trylock_local(lock) spin_trylock(lock)
|
|
--- a/include/linux/spinlock_rt.h
|
|
+++ b/include/linux/spinlock_rt.h
|
|
@@ -18,6 +18,10 @@ do { \
|
|
__rt_spin_lock_init(slock, #slock, &__key); \
|
|
} while (0)
|
|
|
|
+void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock);
|
|
+void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock);
|
|
+int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock);
|
|
+
|
|
extern void __lockfunc rt_spin_lock(spinlock_t *lock);
|
|
extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock);
|
|
extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass);
|
|
@@ -32,20 +36,16 @@ extern int atomic_dec_and_spin_lock(atom
|
|
* lockdep-less calls, for derived types like rwlock:
|
|
* (for trylock they can use rt_mutex_trylock() directly.
|
|
*/
|
|
+extern void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock);
|
|
extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock);
|
|
extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock);
|
|
extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock);
|
|
|
|
-#define spin_lock(lock) \
|
|
- do { \
|
|
- migrate_disable(); \
|
|
- rt_spin_lock(lock); \
|
|
- } while (0)
|
|
+#define spin_lock(lock) rt_spin_lock(lock)
|
|
|
|
#define spin_lock_bh(lock) \
|
|
do { \
|
|
local_bh_disable(); \
|
|
- migrate_disable(); \
|
|
rt_spin_lock(lock); \
|
|
} while (0)
|
|
|
|
@@ -56,24 +56,19 @@ extern int __lockfunc __rt_spin_trylock(
|
|
#define spin_trylock(lock) \
|
|
({ \
|
|
int __locked; \
|
|
- migrate_disable(); \
|
|
__locked = spin_do_trylock(lock); \
|
|
- if (!__locked) \
|
|
- migrate_enable(); \
|
|
__locked; \
|
|
})
|
|
|
|
#ifdef CONFIG_LOCKDEP
|
|
# define spin_lock_nested(lock, subclass) \
|
|
do { \
|
|
- migrate_disable(); \
|
|
rt_spin_lock_nested(lock, subclass); \
|
|
} while (0)
|
|
|
|
#define spin_lock_bh_nested(lock, subclass) \
|
|
do { \
|
|
local_bh_disable(); \
|
|
- migrate_disable(); \
|
|
rt_spin_lock_nested(lock, subclass); \
|
|
} while (0)
|
|
|
|
@@ -81,7 +76,6 @@ extern int __lockfunc __rt_spin_trylock(
|
|
do { \
|
|
typecheck(unsigned long, flags); \
|
|
flags = 0; \
|
|
- migrate_disable(); \
|
|
rt_spin_lock_nested(lock, subclass); \
|
|
} while (0)
|
|
#else
|
|
@@ -117,16 +111,11 @@ static inline unsigned long spin_lock_tr
|
|
/* FIXME: we need rt_spin_lock_nest_lock */
|
|
#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0)
|
|
|
|
-#define spin_unlock(lock) \
|
|
- do { \
|
|
- rt_spin_unlock(lock); \
|
|
- migrate_enable(); \
|
|
- } while (0)
|
|
+#define spin_unlock(lock) rt_spin_unlock(lock)
|
|
|
|
#define spin_unlock_bh(lock) \
|
|
do { \
|
|
rt_spin_unlock(lock); \
|
|
- migrate_enable(); \
|
|
local_bh_enable(); \
|
|
} while (0)
|
|
|
|
--- a/kernel/cpu.c
|
|
+++ b/kernel/cpu.c
|
|
@@ -204,8 +204,8 @@ struct hotplug_pcp {
|
|
};
|
|
|
|
#ifdef CONFIG_PREEMPT_RT_FULL
|
|
-# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock)
|
|
-# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock)
|
|
+# define hotplug_lock(hp) rt_spin_lock__no_mg(&(hp)->lock)
|
|
+# define hotplug_unlock(hp) rt_spin_unlock__no_mg(&(hp)->lock)
|
|
#else
|
|
# define hotplug_lock(hp) mutex_lock(&(hp)->mutex)
|
|
# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex)
|
|
--- a/kernel/locking/lglock.c
|
|
+++ b/kernel/locking/lglock.c
|
|
@@ -10,7 +10,7 @@
|
|
# define lg_do_unlock(l) arch_spin_unlock(l)
|
|
#else
|
|
# define lg_lock_ptr struct rt_mutex
|
|
-# define lg_do_lock(l) __rt_spin_lock(l)
|
|
+# define lg_do_lock(l) __rt_spin_lock__no_mg(l)
|
|
# define lg_do_unlock(l) __rt_spin_unlock(l)
|
|
#endif
|
|
/*
|
|
--- a/kernel/locking/rt.c
|
|
+++ b/kernel/locking/rt.c
|
|
@@ -235,7 +235,6 @@ EXPORT_SYMBOL(rt_read_trylock);
|
|
void __lockfunc rt_write_lock(rwlock_t *rwlock)
|
|
{
|
|
rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
|
|
- migrate_disable();
|
|
__rt_spin_lock(&rwlock->lock);
|
|
}
|
|
EXPORT_SYMBOL(rt_write_lock);
|
|
@@ -249,7 +248,6 @@ void __lockfunc rt_read_lock(rwlock_t *r
|
|
* recursive read locks succeed when current owns the lock
|
|
*/
|
|
if (rt_mutex_owner(lock) != current) {
|
|
- migrate_disable();
|
|
rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
|
|
__rt_spin_lock(lock);
|
|
}
|
|
--- a/kernel/locking/rtmutex.c
|
|
+++ b/kernel/locking/rtmutex.c
|
|
@@ -1103,8 +1103,16 @@ static void noinline __sched rt_spin_lo
|
|
rt_mutex_adjust_prio(current);
|
|
}
|
|
|
|
+void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock)
|
|
+{
|
|
+ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock);
|
|
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
|
|
+}
|
|
+EXPORT_SYMBOL(rt_spin_lock__no_mg);
|
|
+
|
|
void __lockfunc rt_spin_lock(spinlock_t *lock)
|
|
{
|
|
+ migrate_disable();
|
|
rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock);
|
|
spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
|
|
}
|
|
@@ -1112,24 +1120,41 @@ EXPORT_SYMBOL(rt_spin_lock);
|
|
|
|
void __lockfunc __rt_spin_lock(struct rt_mutex *lock)
|
|
{
|
|
+ migrate_disable();
|
|
rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock);
|
|
}
|
|
EXPORT_SYMBOL(__rt_spin_lock);
|
|
|
|
+void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock)
|
|
+{
|
|
+ rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock);
|
|
+}
|
|
+EXPORT_SYMBOL(__rt_spin_lock__no_mg);
|
|
+
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass)
|
|
{
|
|
+ migrate_disable();
|
|
rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock);
|
|
spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
|
|
}
|
|
EXPORT_SYMBOL(rt_spin_lock_nested);
|
|
#endif
|
|
|
|
+void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock)
|
|
+{
|
|
+ /* NOTE: we always pass in '1' for nested, for simplicity */
|
|
+ spin_release(&lock->dep_map, 1, _RET_IP_);
|
|
+ rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
|
|
+}
|
|
+EXPORT_SYMBOL(rt_spin_unlock__no_mg);
|
|
+
|
|
void __lockfunc rt_spin_unlock(spinlock_t *lock)
|
|
{
|
|
/* NOTE: we always pass in '1' for nested, for simplicity */
|
|
spin_release(&lock->dep_map, 1, _RET_IP_);
|
|
rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
|
|
+ migrate_enable();
|
|
}
|
|
EXPORT_SYMBOL(rt_spin_unlock);
|
|
|
|
@@ -1156,12 +1181,27 @@ int __lockfunc __rt_spin_trylock(struct
|
|
return rt_mutex_trylock(lock);
|
|
}
|
|
|
|
+int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock)
|
|
+{
|
|
+ int ret;
|
|
+
|
|
+ ret = rt_mutex_trylock(&lock->lock);
|
|
+ if (ret)
|
|
+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
|
+ return ret;
|
|
+}
|
|
+EXPORT_SYMBOL(rt_spin_trylock__no_mg);
|
|
+
|
|
int __lockfunc rt_spin_trylock(spinlock_t *lock)
|
|
{
|
|
- int ret = rt_mutex_trylock(&lock->lock);
|
|
+ int ret;
|
|
|
|
+ migrate_disable();
|
|
+ ret = rt_mutex_trylock(&lock->lock);
|
|
if (ret)
|
|
spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
|
+ else
|
|
+ migrate_enable();
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(rt_spin_trylock);
|
|
@@ -1200,12 +1240,10 @@ int atomic_dec_and_spin_lock(atomic_t *a
|
|
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
|
|
if (atomic_add_unless(atomic, -1, 1))
|
|
return 0;
|
|
- migrate_disable();
|
|
rt_spin_lock(lock);
|
|
if (atomic_dec_and_test(atomic))
|
|
return 1;
|
|
rt_spin_unlock(lock);
|
|
- migrate_enable();
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(atomic_dec_and_spin_lock);
|