2012-06-17 19:52:06 +00:00
|
|
|
From f079fedbfbd2c15beeca29baaf2de7bd1f04ae0f Mon Sep 17 00:00:00 2001
|
2011-11-20 00:01:55 +00:00
|
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
Date: Wed, 15 Jun 2011 12:36:06 +0200
|
2012-06-17 19:52:06 +00:00
|
|
|
Subject: [138/254] hotplug: Lightweight get online cpus
|
2011-11-20 00:01:55 +00:00
|
|
|
|
|
|
|
get_online_cpus() is a heavy weight function which involves a global
|
|
|
|
mutex. migrate_disable() wants a simpler construct which prevents only
|
|
|
|
a CPU from going doing while a task is in a migrate disabled section.
|
|
|
|
|
|
|
|
Implement a per cpu lockless mechanism, which serializes only in the
|
|
|
|
real unplug case on a global mutex. That serialization affects only
|
|
|
|
tasks on the cpu which should be brought down.
|
|
|
|
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
---
|
2012-06-17 19:52:06 +00:00
|
|
|
include/linux/cpu.h | 4 ++
|
|
|
|
kernel/cpu.c | 127 +++++++++++++++++++++++++++++++++++++++++++++++++--
|
2011-11-20 00:01:55 +00:00
|
|
|
2 files changed, 128 insertions(+), 3 deletions(-)
|
|
|
|
|
2012-06-17 19:52:06 +00:00
|
|
|
diff --git a/include/linux/cpu.h b/include/linux/cpu.h
|
|
|
|
index ee28844..00d2f6f8 100644
|
|
|
|
--- a/include/linux/cpu.h
|
|
|
|
+++ b/include/linux/cpu.h
|
2012-06-04 21:20:09 +00:00
|
|
|
@@ -176,6 +176,8 @@ extern struct bus_type cpu_subsys;
|
2011-11-20 00:01:55 +00:00
|
|
|
|
|
|
|
extern void get_online_cpus(void);
|
|
|
|
extern void put_online_cpus(void);
|
|
|
|
+extern void pin_current_cpu(void);
|
|
|
|
+extern void unpin_current_cpu(void);
|
|
|
|
#define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri)
|
|
|
|
#define register_hotcpu_notifier(nb) register_cpu_notifier(nb)
|
|
|
|
#define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb)
|
2012-06-17 19:52:06 +00:00
|
|
|
@@ -198,6 +200,8 @@ static inline void cpu_hotplug_driver_unlock(void)
|
2011-11-20 00:01:55 +00:00
|
|
|
|
|
|
|
#define get_online_cpus() do { } while (0)
|
|
|
|
#define put_online_cpus() do { } while (0)
|
|
|
|
+static inline void pin_current_cpu(void) { }
|
|
|
|
+static inline void unpin_current_cpu(void) { }
|
|
|
|
#define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
|
|
|
|
/* These aren't inline functions due to a GCC bug. */
|
|
|
|
#define register_hotcpu_notifier(nb) ({ (void)(nb); 0; })
|
2012-06-17 19:52:06 +00:00
|
|
|
diff --git a/kernel/cpu.c b/kernel/cpu.c
|
|
|
|
index 2060c6e..8aaa210 100644
|
|
|
|
--- a/kernel/cpu.c
|
|
|
|
+++ b/kernel/cpu.c
|
2011-11-20 00:01:55 +00:00
|
|
|
@@ -58,6 +58,102 @@ static struct {
|
|
|
|
.refcount = 0,
|
|
|
|
};
|
|
|
|
|
|
|
|
+struct hotplug_pcp {
|
|
|
|
+ struct task_struct *unplug;
|
|
|
|
+ int refcount;
|
|
|
|
+ struct completion synced;
|
|
|
|
+};
|
|
|
|
+
|
|
|
|
+static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp);
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * pin_current_cpu - Prevent the current cpu from being unplugged
|
|
|
|
+ *
|
|
|
|
+ * Lightweight version of get_online_cpus() to prevent cpu from being
|
|
|
|
+ * unplugged when code runs in a migration disabled region.
|
|
|
|
+ *
|
|
|
|
+ * Must be called with preemption disabled (preempt_count = 1)!
|
|
|
|
+ */
|
|
|
|
+void pin_current_cpu(void)
|
|
|
|
+{
|
|
|
|
+ struct hotplug_pcp *hp = &__get_cpu_var(hotplug_pcp);
|
|
|
|
+
|
|
|
|
+retry:
|
|
|
|
+ if (!hp->unplug || hp->refcount || preempt_count() > 1 ||
|
|
|
|
+ hp->unplug == current || (current->flags & PF_STOMPER)) {
|
|
|
|
+ hp->refcount++;
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+ preempt_enable();
|
|
|
|
+ mutex_lock(&cpu_hotplug.lock);
|
|
|
|
+ mutex_unlock(&cpu_hotplug.lock);
|
|
|
|
+ preempt_disable();
|
|
|
|
+ goto retry;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * unpin_current_cpu - Allow unplug of current cpu
|
|
|
|
+ *
|
|
|
|
+ * Must be called with preemption or interrupts disabled!
|
|
|
|
+ */
|
|
|
|
+void unpin_current_cpu(void)
|
|
|
|
+{
|
|
|
|
+ struct hotplug_pcp *hp = &__get_cpu_var(hotplug_pcp);
|
|
|
|
+
|
|
|
|
+ WARN_ON(hp->refcount <= 0);
|
|
|
|
+
|
|
|
|
+ /* This is safe. sync_unplug_thread is pinned to this cpu */
|
|
|
|
+ if (!--hp->refcount && hp->unplug && hp->unplug != current &&
|
|
|
|
+ !(current->flags & PF_STOMPER))
|
|
|
|
+ wake_up_process(hp->unplug);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * FIXME: Is this really correct under all circumstances ?
|
|
|
|
+ */
|
|
|
|
+static int sync_unplug_thread(void *data)
|
|
|
|
+{
|
|
|
|
+ struct hotplug_pcp *hp = data;
|
|
|
|
+
|
|
|
|
+ preempt_disable();
|
|
|
|
+ hp->unplug = current;
|
|
|
|
+ set_current_state(TASK_UNINTERRUPTIBLE);
|
|
|
|
+ while (hp->refcount) {
|
|
|
|
+ schedule_preempt_disabled();
|
|
|
|
+ set_current_state(TASK_UNINTERRUPTIBLE);
|
|
|
|
+ }
|
|
|
|
+ set_current_state(TASK_RUNNING);
|
|
|
|
+ preempt_enable();
|
|
|
|
+ complete(&hp->synced);
|
|
|
|
+ return 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * Start the sync_unplug_thread on the target cpu and wait for it to
|
|
|
|
+ * complete.
|
|
|
|
+ */
|
|
|
|
+static int cpu_unplug_begin(unsigned int cpu)
|
|
|
|
+{
|
|
|
|
+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
|
|
|
|
+ struct task_struct *tsk;
|
|
|
|
+
|
|
|
|
+ init_completion(&hp->synced);
|
|
|
|
+ tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d\n", cpu);
|
|
|
|
+ if (IS_ERR(tsk))
|
|
|
|
+ return (PTR_ERR(tsk));
|
|
|
|
+ kthread_bind(tsk, cpu);
|
|
|
|
+ wake_up_process(tsk);
|
|
|
|
+ wait_for_completion(&hp->synced);
|
|
|
|
+ return 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static void cpu_unplug_done(unsigned int cpu)
|
|
|
|
+{
|
|
|
|
+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
|
|
|
|
+
|
|
|
|
+ hp->unplug = NULL;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
void get_online_cpus(void)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2012-06-17 19:52:06 +00:00
|
|
|
@@ -210,13 +306,14 @@ static int __ref take_cpu_down(void *_param)
|
2011-11-20 00:01:55 +00:00
|
|
|
/* Requires cpu_add_remove_lock to be held */
|
|
|
|
static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
|
|
|
|
{
|
|
|
|
- int err, nr_calls = 0;
|
|
|
|
+ int mycpu, err, nr_calls = 0;
|
|
|
|
void *hcpu = (void *)(long)cpu;
|
|
|
|
unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
|
|
|
|
struct take_cpu_down_param tcd_param = {
|
|
|
|
.mod = mod,
|
|
|
|
.hcpu = hcpu,
|
|
|
|
};
|
|
|
|
+ cpumask_var_t cpumask;
|
|
|
|
|
|
|
|
if (num_online_cpus() == 1)
|
|
|
|
return -EBUSY;
|
2012-06-17 19:52:06 +00:00
|
|
|
@@ -224,7 +321,20 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
|
2011-11-20 00:01:55 +00:00
|
|
|
if (!cpu_online(cpu))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
- cpu_hotplug_begin();
|
|
|
|
+ /* Move the downtaker off the unplug cpu */
|
|
|
|
+ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL))
|
|
|
|
+ return -ENOMEM;
|
|
|
|
+ cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu));
|
|
|
|
+ set_cpus_allowed_ptr(current, cpumask);
|
|
|
|
+ free_cpumask_var(cpumask);
|
|
|
|
+ preempt_disable();
|
|
|
|
+ mycpu = smp_processor_id();
|
|
|
|
+ if (mycpu == cpu) {
|
|
|
|
+ printk(KERN_ERR "Yuck! Still on unplug CPU\n!");
|
|
|
|
+ preempt_enable();
|
|
|
|
+ return -EBUSY;
|
|
|
|
+ }
|
|
|
|
+ preempt_enable();
|
|
|
|
|
|
|
|
err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
|
|
|
|
if (err) {
|
2012-06-17 19:52:06 +00:00
|
|
|
@@ -232,7 +342,16 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
|
2011-11-20 00:01:55 +00:00
|
|
|
__cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
|
|
|
|
printk("%s: attempt to take down CPU %u failed\n",
|
|
|
|
__func__, cpu);
|
|
|
|
- goto out_release;
|
|
|
|
+ goto out_cancel;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ cpu_hotplug_begin();
|
|
|
|
+ err = cpu_unplug_begin(cpu);
|
|
|
|
+ if (err) {
|
|
|
|
+ nr_calls--;
|
|
|
|
+ __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
|
|
|
|
+ printk("cpu_unplug_begin(%d) failed\n", cpu);
|
|
|
|
+ goto out_cancel;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
|
2012-06-17 19:52:06 +00:00
|
|
|
@@ -263,6 +382,8 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
|
2011-11-20 00:01:55 +00:00
|
|
|
check_for_tasks(cpu);
|
|
|
|
|
|
|
|
out_release:
|
|
|
|
+ cpu_unplug_done(cpu);
|
|
|
|
+out_cancel:
|
|
|
|
cpu_hotplug_done();
|
|
|
|
if (!err)
|
|
|
|
cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
|