219 lines
6.2 KiB
Diff
219 lines
6.2 KiB
Diff
Subject: wait-simple: Rework for use with completions
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
Date: Thu, 10 Jan 2013 11:47:35 +0100
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.0/patches-4.0.5-rt3.tar.xz
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
---
|
|
include/linux/wait-simple.h | 60 +++++++-----------------------------
|
|
kernel/sched/wait-simple.c | 73 ++++++++++++++++++++++++++++++++++++++------
|
|
2 files changed, 76 insertions(+), 57 deletions(-)
|
|
|
|
--- a/include/linux/wait-simple.h
|
|
+++ b/include/linux/wait-simple.h
|
|
@@ -22,12 +22,14 @@ struct swait_head {
|
|
struct list_head list;
|
|
};
|
|
|
|
-#define DEFINE_SWAIT_HEAD(name) \
|
|
- struct swait_head name = { \
|
|
+#define SWAIT_HEAD_INITIALIZER(name) { \
|
|
.lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \
|
|
.list = LIST_HEAD_INIT((name).list), \
|
|
}
|
|
|
|
+#define DEFINE_SWAIT_HEAD(name) \
|
|
+ struct swait_head name = SWAIT_HEAD_INITIALIZER(name)
|
|
+
|
|
extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key);
|
|
|
|
#define init_swait_head(swh) \
|
|
@@ -40,63 +42,25 @@ extern void __init_swait_head(struct swa
|
|
/*
|
|
* Waiter functions
|
|
*/
|
|
-static inline bool swaiter_enqueued(struct swaiter *w)
|
|
-{
|
|
- return w->task != NULL;
|
|
-}
|
|
-
|
|
+extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w);
|
|
extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state);
|
|
+extern void swait_finish_locked(struct swait_head *head, struct swaiter *w);
|
|
extern void swait_finish(struct swait_head *head, struct swaiter *w);
|
|
|
|
/*
|
|
- * Adds w to head->list. Must be called with head->lock locked.
|
|
- */
|
|
-static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w)
|
|
-{
|
|
- list_add(&w->node, &head->list);
|
|
- /* We can't let the condition leak before the setting of head */
|
|
- smp_mb();
|
|
-}
|
|
-
|
|
-/*
|
|
- * Removes w from head->list. Must be called with head->lock locked.
|
|
- */
|
|
-static inline void __swait_dequeue(struct swaiter *w)
|
|
-{
|
|
- list_del_init(&w->node);
|
|
-}
|
|
-
|
|
-/*
|
|
- * Check whether a head has waiters enqueued
|
|
- */
|
|
-static inline bool swait_head_has_waiters(struct swait_head *h)
|
|
-{
|
|
- /* Make sure the condition is visible before checking list_empty() */
|
|
- smp_mb();
|
|
- return !list_empty(&h->list);
|
|
-}
|
|
-
|
|
-/*
|
|
* Wakeup functions
|
|
*/
|
|
-extern int __swait_wake(struct swait_head *head, unsigned int state);
|
|
+extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num);
|
|
+extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num);
|
|
|
|
-static inline int swait_wake(struct swait_head *head)
|
|
-{
|
|
- return swait_head_has_waiters(head) ?
|
|
- __swait_wake(head, TASK_NORMAL) : 0;
|
|
-}
|
|
-
|
|
-static inline int swait_wake_interruptible(struct swait_head *head)
|
|
-{
|
|
- return swait_head_has_waiters(head) ?
|
|
- __swait_wake(head, TASK_INTERRUPTIBLE) : 0;
|
|
-}
|
|
+#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1)
|
|
+#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1)
|
|
+#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0)
|
|
+#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0)
|
|
|
|
/*
|
|
* Event API
|
|
*/
|
|
-
|
|
#define __swait_event(wq, condition) \
|
|
do { \
|
|
DEFINE_SWAITER(__wait); \
|
|
--- a/kernel/sched/wait-simple.c
|
|
+++ b/kernel/sched/wait-simple.c
|
|
@@ -12,6 +12,28 @@
|
|
#include <linux/sched.h>
|
|
#include <linux/wait-simple.h>
|
|
|
|
+/* Adds w to head->list. Must be called with head->lock locked. */
|
|
+static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w)
|
|
+{
|
|
+ list_add(&w->node, &head->list);
|
|
+ /* We can't let the condition leak before the setting of head */
|
|
+ smp_mb();
|
|
+}
|
|
+
|
|
+/* Removes w from head->list. Must be called with head->lock locked. */
|
|
+static inline void __swait_dequeue(struct swaiter *w)
|
|
+{
|
|
+ list_del_init(&w->node);
|
|
+}
|
|
+
|
|
+/* Check whether a head has waiters enqueued */
|
|
+static inline bool swait_head_has_waiters(struct swait_head *h)
|
|
+{
|
|
+ /* Make sure the condition is visible before checking list_empty() */
|
|
+ smp_mb();
|
|
+ return !list_empty(&h->list);
|
|
+}
|
|
+
|
|
void __init_swait_head(struct swait_head *head, struct lock_class_key *key)
|
|
{
|
|
raw_spin_lock_init(&head->lock);
|
|
@@ -20,19 +42,31 @@ void __init_swait_head(struct swait_head
|
|
}
|
|
EXPORT_SYMBOL(__init_swait_head);
|
|
|
|
+void swait_prepare_locked(struct swait_head *head, struct swaiter *w)
|
|
+{
|
|
+ w->task = current;
|
|
+ if (list_empty(&w->node))
|
|
+ __swait_enqueue(head, w);
|
|
+}
|
|
+
|
|
void swait_prepare(struct swait_head *head, struct swaiter *w, int state)
|
|
{
|
|
unsigned long flags;
|
|
|
|
raw_spin_lock_irqsave(&head->lock, flags);
|
|
- w->task = current;
|
|
- if (list_empty(&w->node))
|
|
- __swait_enqueue(head, w);
|
|
- set_current_state(state);
|
|
+ swait_prepare_locked(head, w);
|
|
+ __set_current_state(state);
|
|
raw_spin_unlock_irqrestore(&head->lock, flags);
|
|
}
|
|
EXPORT_SYMBOL(swait_prepare);
|
|
|
|
+void swait_finish_locked(struct swait_head *head, struct swaiter *w)
|
|
+{
|
|
+ __set_current_state(TASK_RUNNING);
|
|
+ if (w->task)
|
|
+ __swait_dequeue(w);
|
|
+}
|
|
+
|
|
void swait_finish(struct swait_head *head, struct swaiter *w)
|
|
{
|
|
unsigned long flags;
|
|
@@ -46,22 +80,43 @@ void swait_finish(struct swait_head *hea
|
|
}
|
|
EXPORT_SYMBOL(swait_finish);
|
|
|
|
-int __swait_wake(struct swait_head *head, unsigned int state)
|
|
+unsigned int
|
|
+__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num)
|
|
{
|
|
struct swaiter *curr, *next;
|
|
- unsigned long flags;
|
|
int woken = 0;
|
|
|
|
- raw_spin_lock_irqsave(&head->lock, flags);
|
|
-
|
|
list_for_each_entry_safe(curr, next, &head->list, node) {
|
|
if (wake_up_state(curr->task, state)) {
|
|
__swait_dequeue(curr);
|
|
+ /*
|
|
+ * The waiting task can free the waiter as
|
|
+ * soon as curr->task = NULL is written,
|
|
+ * without taking any locks. A memory barrier
|
|
+ * is required here to prevent the following
|
|
+ * store to curr->task from getting ahead of
|
|
+ * the dequeue operation.
|
|
+ */
|
|
+ smp_wmb();
|
|
curr->task = NULL;
|
|
- woken++;
|
|
+ if (++woken == num)
|
|
+ break;
|
|
}
|
|
}
|
|
+ return woken;
|
|
+}
|
|
+
|
|
+unsigned int
|
|
+__swait_wake(struct swait_head *head, unsigned int state, unsigned int num)
|
|
+{
|
|
+ unsigned long flags;
|
|
+ int woken;
|
|
|
|
+ if (!swait_head_has_waiters(head))
|
|
+ return 0;
|
|
+
|
|
+ raw_spin_lock_irqsave(&head->lock, flags);
|
|
+ woken = __swait_wake_locked(head, state, num);
|
|
raw_spin_unlock_irqrestore(&head->lock, flags);
|
|
return woken;
|
|
}
|