[PATCH 09/89] sched/wake_q: Move the wake-queue types and interfaces from sched.h into <linux/sched/wake_q.h>
From: Ingo Molnar
Date: Mon Feb 06 2017 - 08:56:17 EST
Note that this requires the basic task->wake_q type to be an opaque void *.
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Mike Galbraith <efault@xxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: linux-kernel@xxxxxxxxxxxxxxx
Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
---
include/linux/sched.h | 52 +---------------------------------------------------
include/linux/sched/wake_q.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
ipc/mqueue.c | 1 +
ipc/msg.c | 1 +
ipc/sem.c | 1 +
kernel/fork.c | 2 +-
kernel/futex.c | 1 +
kernel/locking/mutex.c | 1 +
kernel/locking/rtmutex.c | 1 +
kernel/locking/rtmutex_common.h | 1 +
kernel/locking/rwsem-xadd.c | 3 ++-
kernel/sched/core.c | 6 +++---
kernel/sched/sched.h | 1 +
13 files changed, 69 insertions(+), 56 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index ebc11bb42c49..e973a0f5550c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -961,56 +961,6 @@ void force_schedstat_enabled(void);
# define SCHED_FIXEDPOINT_SHIFT 10
# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
-/*
- * Wake-queues are lists of tasks with a pending wakeup, whose
- * callers have already marked the task as woken internally,
- * and can thus carry on. A common use case is being able to
- * do the wakeups once the corresponding user lock as been
- * released.
- *
- * We hold reference to each task in the list across the wakeup,
- * thus guaranteeing that the memory is still valid by the time
- * the actual wakeups are performed in wake_up_q().
- *
- * One per task suffices, because there's never a need for a task to be
- * in two wake queues simultaneously; it is forbidden to abandon a task
- * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
- * already in a wake queue, the wakeup will happen soon and the second
- * waker can just skip it.
- *
- * The DEFINE_WAKE_Q macro declares and initializes the list head.
- * wake_up_q() does NOT reinitialize the list; it's expected to be
- * called near the end of a function. Otherwise, the list can be
- * re-initialized for later re-use by wake_q_init().
- *
- * Note that this can cause spurious wakeups. schedule() callers
- * must ensure the call is done inside a loop, confirming that the
- * wakeup condition has in fact occurred.
- */
-struct wake_q_node {
- struct wake_q_node *next;
-};
-
-struct wake_q_head {
- struct wake_q_node *first;
- struct wake_q_node **lastp;
-};
-
-#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
-
-#define DEFINE_WAKE_Q(name) \
- struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
-
-static inline void wake_q_init(struct wake_q_head *head)
-{
- head->first = WAKE_Q_TAIL;
- head->lastp = &head->first;
-}
-
-extern void wake_q_add(struct wake_q_head *head,
- struct task_struct *task);
-extern void wake_up_q(struct wake_q_head *head);
-
struct io_context; /* See blkdev.h */
@@ -1507,7 +1457,7 @@ struct task_struct {
/* Protection of the PI data structures: */
raw_spinlock_t pi_lock;
- struct wake_q_node wake_q;
+ void *wake_q;
#ifdef CONFIG_RT_MUTEXES
/* PI waiters blocked on a rt_mutex held by this task */
diff --git a/include/linux/sched/wake_q.h b/include/linux/sched/wake_q.h
new file mode 100644
index 000000000000..9a32f17b6a5e
--- /dev/null
+++ b/include/linux/sched/wake_q.h
@@ -0,0 +1,54 @@
+#ifndef _LINUX_SCHED_WAKE_Q_H
+#define _LINUX_SCHED_WAKE_Q_H
+
+/*
+ * Wake-queues are lists of tasks with a pending wakeup, whose
+ * callers have already marked the task as woken internally,
+ * and can thus carry on. A common use case is being able to
+ * do the wakeups once the corresponding user lock as been
+ * released.
+ *
+ * We hold reference to each task in the list across the wakeup,
+ * thus guaranteeing that the memory is still valid by the time
+ * the actual wakeups are performed in wake_up_q().
+ *
+ * One per task suffices, because there's never a need for a task to be
+ * in two wake queues simultaneously; it is forbidden to abandon a task
+ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
+ * already in a wake queue, the wakeup will happen soon and the second
+ * waker can just skip it.
+ *
+ * The DEFINE_WAKE_Q macro declares and initializes the list head.
+ * wake_up_q() does NOT reinitialize the list; it's expected to be
+ * called near the end of a function. Otherwise, the list can be
+ * re-initialized for later re-use by wake_q_init().
+ *
+ * Note that this can cause spurious wakeups. schedule() callers
+ * must ensure the call is done inside a loop, confirming that the
+ * wakeup condition has in fact occurred.
+ */
+struct wake_q_node {
+ struct wake_q_node *next;
+};
+
+struct wake_q_head {
+ struct wake_q_node *first;
+ struct wake_q_node **lastp;
+};
+
+#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
+
+#define DEFINE_WAKE_Q(name) \
+ struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
+
+static inline void wake_q_init(struct wake_q_head *head)
+{
+ head->first = WAKE_Q_TAIL;
+ head->lastp = &head->first;
+}
+
+extern void wake_q_add(struct wake_q_head *head,
+ struct task_struct *task);
+extern void wake_up_q(struct wake_q_head *head);
+
+#endif /* _LINUX_SCHED_WAKE_Q_H */
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 7a2d8f0c8ae5..a604f1036067 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -35,6 +35,7 @@
#include <linux/ipc_namespace.h>
#include <linux/user_namespace.h>
#include <linux/slab.h>
+#include <linux/sched/wake_q.h>
#include <net/sock.h>
#include "util.h"
diff --git a/ipc/msg.c b/ipc/msg.c
index e3e52ce01123..ecc387e573f6 100644
--- a/ipc/msg.c
+++ b/ipc/msg.c
@@ -31,6 +31,7 @@
#include <linux/list.h>
#include <linux/security.h>
#include <linux/sched.h>
+#include <linux/sched/wake_q.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/seq_file.h>
diff --git a/ipc/sem.c b/ipc/sem.c
index 3ec5742b5640..c65dff526a9f 100644
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -82,6 +82,7 @@
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/ipc_namespace.h>
+#include <linux/sched/wake_q.h>
#include <linux/uaccess.h>
#include "util.h"
diff --git a/kernel/fork.c b/kernel/fork.c
index 077ce3b4c0ef..9aa2ca28a76c 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -538,7 +538,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
tsk->splice_pipe = NULL;
tsk->task_frag.page = NULL;
- tsk->wake_q.next = NULL;
+ tsk->wake_q = NULL;
account_kernel_stack(tsk, 1);
diff --git a/kernel/futex.c b/kernel/futex.c
index 0842c8ca534b..e41472f33c73 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -61,6 +61,7 @@
#include <linux/nsproxy.h>
#include <linux/ptrace.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
#include <linux/hugetlb.h>
#include <linux/freezer.h>
#include <linux/bootmem.h>
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index ad2d9e22697b..57f6311e2405 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -21,6 +21,7 @@
#include <linux/ww_mutex.h>
#include <linux/sched.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index d340be3a488f..d4f798491361 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -15,6 +15,7 @@
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
+#include <linux/sched/wake_q.h>
#include <linux/timer.h>
#include "rtmutex_common.h"
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index 990134617b4c..856dfff5c33a 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -13,6 +13,7 @@
#define __KERNEL_RTMUTEX_COMMON_H
#include <linux/rtmutex.h>
+#include <linux/sched/wake_q.h>
/*
* This is the control structure for tasks blocked on a rt_mutex,
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index 2ad8d8dc3bb1..4fe8d8ad4396 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -10,10 +10,11 @@
* and Davidlohr Bueso <davidlohr@xxxxxx>. Based on mutexes.
*/
#include <linux/rwsem.h>
-#include <linux/sched.h>
#include <linux/init.h>
#include <linux/export.h>
+#include <linux/sched.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
#include <linux/osq_lock.h>
#include "rwsem.h"
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e4aa470ed454..fed9625fa954 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -426,7 +426,7 @@ static bool set_nr_if_polling(struct task_struct *p)
void wake_q_add(struct wake_q_head *head, struct task_struct *task)
{
- struct wake_q_node *node = &task->wake_q;
+ struct wake_q_node *node = (void *)&task->wake_q;
/*
* Atomically grab the task, if ->wake_q is !nil already it means
@@ -455,11 +455,11 @@ void wake_up_q(struct wake_q_head *head)
while (node != WAKE_Q_TAIL) {
struct task_struct *task;
- task = container_of(node, struct task_struct, wake_q);
+ task = container_of((void *)node, struct task_struct, wake_q);
BUG_ON(!task);
/* Task can safely be re-inserted now: */
node = node->next;
- task->wake_q.next = NULL;
+ task->wake_q = NULL;
/*
* wake_up_process() implies a wmb() to pair with the queueing
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 319fcf80930c..ebb66cdeff70 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3,6 +3,7 @@
#include <linux/sched/sysctl.h>
#include <linux/sched/topology.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
#include <linux/u64_stats_sync.h>
#include <linux/sched/deadline.h>
#include <linux/kernel_stat.h>
--
2.7.4