188 lines
6.3 KiB
Diff
188 lines
6.3 KiB
Diff
From 36ff989b9512c60087e23ab922f45cede462a4ad Mon Sep 17 00:00:00 2001
|
|
From: Peter Zijlstra <a.p.zijlstra@chello.nl>
|
|
Date: Thu, 11 Aug 2011 15:14:58 +0200
|
|
Subject: [PATCH 169/279] sched: Generic migrate_disable
|
|
|
|
Make migrate_disable() be a preempt_disable() for !rt kernels. This
|
|
allows generic code to use it but still enforces that these code
|
|
sections stay relatively small.
|
|
|
|
A preemptible migrate_disable() accessible for general use would allow
|
|
people growing arbitrary per-cpu crap instead of clean these things
|
|
up.
|
|
|
|
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
|
|
Link: http://lkml.kernel.org/n/tip-275i87sl8e1jcamtchmehonm@git.kernel.org
|
|
---
|
|
include/linux/preempt.h | 21 +++++++++------------
|
|
include/linux/sched.h | 13 +++++++++++++
|
|
include/linux/smp.h | 9 ++-------
|
|
kernel/sched.c | 6 ++++--
|
|
kernel/trace/trace.c | 2 +-
|
|
lib/smp_processor_id.c | 2 +-
|
|
6 files changed, 30 insertions(+), 23 deletions(-)
|
|
|
|
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
|
|
index 5aa7916..6450c01 100644
|
|
--- a/include/linux/preempt.h
|
|
+++ b/include/linux/preempt.h
|
|
@@ -108,28 +108,25 @@ do { \
|
|
|
|
#endif /* CONFIG_PREEMPT_COUNT */
|
|
|
|
-#ifdef CONFIG_SMP
|
|
-extern void migrate_disable(void);
|
|
-extern void migrate_enable(void);
|
|
-#else
|
|
-# define migrate_disable() do { } while (0)
|
|
-# define migrate_enable() do { } while (0)
|
|
-#endif
|
|
-
|
|
#ifdef CONFIG_PREEMPT_RT_FULL
|
|
# define preempt_disable_rt() preempt_disable()
|
|
# define preempt_enable_rt() preempt_enable()
|
|
# define preempt_disable_nort() do { } while (0)
|
|
# define preempt_enable_nort() do { } while (0)
|
|
-# define migrate_disable_rt() migrate_disable()
|
|
-# define migrate_enable_rt() migrate_enable()
|
|
+# ifdef CONFIG_SMP
|
|
+ extern void migrate_disable(void);
|
|
+ extern void migrate_enable(void);
|
|
+# else /* CONFIG_SMP */
|
|
+# define migrate_disable() do { } while (0)
|
|
+# define migrate_enable() do { } while (0)
|
|
+# endif /* CONFIG_SMP */
|
|
#else
|
|
# define preempt_disable_rt() do { } while (0)
|
|
# define preempt_enable_rt() do { } while (0)
|
|
# define preempt_disable_nort() preempt_disable()
|
|
# define preempt_enable_nort() preempt_enable()
|
|
-# define migrate_disable_rt() do { } while (0)
|
|
-# define migrate_enable_rt() do { } while (0)
|
|
+# define migrate_disable() preempt_disable()
|
|
+# define migrate_enable() preempt_enable()
|
|
#endif
|
|
|
|
#ifdef CONFIG_PREEMPT_NOTIFIERS
|
|
diff --git a/include/linux/sched.h b/include/linux/sched.h
|
|
index 3e8b05e..b409b94 100644
|
|
--- a/include/linux/sched.h
|
|
+++ b/include/linux/sched.h
|
|
@@ -1263,7 +1263,9 @@ struct task_struct {
|
|
#endif
|
|
|
|
unsigned int policy;
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
int migrate_disable;
|
|
+#endif
|
|
cpumask_t cpus_allowed;
|
|
|
|
#ifdef CONFIG_PREEMPT_RCU
|
|
@@ -2693,11 +2695,22 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
+static inline int __migrate_disabled(struct task_struct *p)
|
|
+{
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+ return p->migrate_disable;
|
|
+#else
|
|
+ return 0;
|
|
+#endif
|
|
+}
|
|
+
|
|
/* Future-safe accessor for struct task_struct's cpus_allowed. */
|
|
static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p)
|
|
{
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
if (p->migrate_disable)
|
|
return cpumask_of(task_cpu(p));
|
|
+#endif
|
|
|
|
return &p->cpus_allowed;
|
|
}
|
|
diff --git a/include/linux/smp.h b/include/linux/smp.h
|
|
index 94c8430..78fd0a2 100644
|
|
--- a/include/linux/smp.h
|
|
+++ b/include/linux/smp.h
|
|
@@ -172,13 +172,8 @@ smp_call_function_any(const struct cpumask *mask, smp_call_func_t func,
|
|
#define get_cpu() ({ preempt_disable(); smp_processor_id(); })
|
|
#define put_cpu() preempt_enable()
|
|
|
|
-#ifndef CONFIG_PREEMPT_RT_FULL
|
|
-# define get_cpu_light() get_cpu()
|
|
-# define put_cpu_light() put_cpu()
|
|
-#else
|
|
-# define get_cpu_light() ({ migrate_disable(); smp_processor_id(); })
|
|
-# define put_cpu_light() migrate_enable()
|
|
-#endif
|
|
+#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); })
|
|
+#define put_cpu_light() migrate_enable()
|
|
|
|
/*
|
|
* Callback to arch code if there's nosmp or maxcpus=0 on the
|
|
diff --git a/kernel/sched.c b/kernel/sched.c
|
|
index 45d61e7..b404fbf 100644
|
|
--- a/kernel/sched.c
|
|
+++ b/kernel/sched.c
|
|
@@ -6416,7 +6416,7 @@ static inline void sched_init_granularity(void)
|
|
#ifdef CONFIG_SMP
|
|
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
|
|
{
|
|
- if (!p->migrate_disable) {
|
|
+ if (!__migrate_disabled(p)) {
|
|
if (p->sched_class && p->sched_class->set_cpus_allowed)
|
|
p->sched_class->set_cpus_allowed(p, new_mask);
|
|
p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
|
|
@@ -6472,7 +6472,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
|
|
do_set_cpus_allowed(p, new_mask);
|
|
|
|
/* Can the task run on the task's current CPU? If so, we're done */
|
|
- if (cpumask_test_cpu(task_cpu(p), new_mask) || p->migrate_disable)
|
|
+ if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p))
|
|
goto out;
|
|
|
|
dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
|
|
@@ -6491,6 +6491,7 @@ out:
|
|
}
|
|
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
void migrate_disable(void)
|
|
{
|
|
struct task_struct *p = current;
|
|
@@ -6583,6 +6584,7 @@ void migrate_enable(void)
|
|
preempt_enable();
|
|
}
|
|
EXPORT_SYMBOL_GPL(migrate_enable);
|
|
+#endif /* CONFIG_PREEMPT_RT_FULL */
|
|
|
|
/*
|
|
* Move (not current) task off this cpu, onto dest cpu. We're doing
|
|
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
|
|
index 1ea52d8..8d7ee39 100644
|
|
--- a/kernel/trace/trace.c
|
|
+++ b/kernel/trace/trace.c
|
|
@@ -1124,7 +1124,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
|
|
((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
|
|
(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
|
|
|
|
- entry->migrate_disable = (tsk) ? tsk->migrate_disable & 0xFF : 0;
|
|
+ entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
|
|
|
|
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
|
|
index 60a7569..b5e9241 100644
|
|
--- a/lib/smp_processor_id.c
|
|
+++ b/lib/smp_processor_id.c
|
|
@@ -41,7 +41,7 @@ notrace unsigned int debug_smp_processor_id(void)
|
|
|
|
printk(KERN_ERR "BUG: using smp_processor_id() in preemptible [%08x %08x] "
|
|
"code: %s/%d\n", preempt_count() - 1,
|
|
- current->migrate_disable, current->comm, current->pid);
|
|
+ __migrate_disabled(current), current->comm, current->pid);
|
|
print_symbol("caller is %s\n", (long)__builtin_return_address(0));
|
|
dump_stack();
|
|
|