2020-04-21 20:13:52 +00:00
|
|
|
From 3e6511a5a97439410e2888198a743dfe4162bcbc Mon Sep 17 00:00:00 2001
|
2018-08-27 14:32:32 +00:00
|
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
2019-04-08 23:49:20 +00:00
|
|
|
Date: Sun, 13 Nov 2011 17:17:09 +0100
|
2020-04-21 20:13:52 +00:00
|
|
|
Subject: [PATCH 119/325] softirq: Check preemption after reenabling interrupts
|
|
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.115-rt48.tar.xz
|
2018-08-27 14:32:32 +00:00
|
|
|
|
|
|
|
raise_softirq_irqoff() disables interrupts and wakes the softirq
|
|
|
|
daemon, but after reenabling interrupts there is no preemption check,
|
|
|
|
so the execution of the softirq thread might be delayed arbitrarily.
|
|
|
|
|
|
|
|
In principle we could add that check to local_irq_enable/restore, but
|
|
|
|
that's overkill as the rasie_softirq_irqoff() sections are the only
|
|
|
|
ones which show this behaviour.
|
|
|
|
|
|
|
|
Reported-by: Carsten Emde <cbe@osadl.org>
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
---
|
2019-04-08 23:49:20 +00:00
|
|
|
block/blk-softirq.c | 3 +++
|
|
|
|
include/linux/preempt.h | 3 +++
|
|
|
|
lib/irq_poll.c | 5 +++++
|
|
|
|
net/core/dev.c | 7 +++++++
|
2018-08-27 14:32:32 +00:00
|
|
|
4 files changed, 18 insertions(+)
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
|
|
|
|
index 15c1f5e12eb8..1628277885a1 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/block/blk-softirq.c
|
|
|
|
+++ b/block/blk-softirq.c
|
|
|
|
@@ -53,6 +53,7 @@ static void trigger_softirq(void *data)
|
|
|
|
raise_softirq_irqoff(BLOCK_SOFTIRQ);
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -91,6 +92,7 @@ static int blk_softirq_cpu_dead(unsigned int cpu)
|
2018-08-27 14:32:32 +00:00
|
|
|
this_cpu_ptr(&blk_cpu_done));
|
|
|
|
raise_softirq_irqoff(BLOCK_SOFTIRQ);
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -143,6 +145,7 @@ void __blk_complete_request(struct request *req)
|
2018-08-27 14:32:32 +00:00
|
|
|
goto do_local;
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__blk_complete_request);
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
|
|
|
|
index b7fe717eb1f4..9984f2b75b73 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/include/linux/preempt.h
|
|
|
|
+++ b/include/linux/preempt.h
|
|
|
|
@@ -187,8 +187,10 @@ do { \
|
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT_RT_BASE
|
|
|
|
# define preempt_enable_no_resched() sched_preempt_enable_no_resched()
|
|
|
|
+# define preempt_check_resched_rt() preempt_check_resched()
|
|
|
|
#else
|
|
|
|
# define preempt_enable_no_resched() preempt_enable()
|
|
|
|
+# define preempt_check_resched_rt() barrier();
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define preemptible() (preempt_count() == 0 && !irqs_disabled())
|
|
|
|
@@ -275,6 +277,7 @@ do { \
|
|
|
|
#define preempt_disable_notrace() barrier()
|
|
|
|
#define preempt_enable_no_resched_notrace() barrier()
|
|
|
|
#define preempt_enable_notrace() barrier()
|
|
|
|
+#define preempt_check_resched_rt() barrier()
|
|
|
|
#define preemptible() 0
|
|
|
|
|
|
|
|
#define migrate_disable() barrier()
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/lib/irq_poll.c b/lib/irq_poll.c
|
|
|
|
index 86a709954f5a..9c069ef83d6d 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/lib/irq_poll.c
|
|
|
|
+++ b/lib/irq_poll.c
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -37,6 +37,7 @@ void irq_poll_sched(struct irq_poll *iop)
|
2018-08-27 14:32:32 +00:00
|
|
|
list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
|
|
|
|
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(irq_poll_sched);
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -72,6 +73,7 @@ void irq_poll_complete(struct irq_poll *iop)
|
2018-08-27 14:32:32 +00:00
|
|
|
local_irq_save(flags);
|
|
|
|
__irq_poll_complete(iop);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(irq_poll_complete);
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -96,6 +98,7 @@ static void __latent_entropy irq_poll_softirq(struct softirq_action *h)
|
2018-08-27 14:32:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
/* Even though interrupts have been re-enabled, this
|
|
|
|
* access is safe because interrupts can only add new
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -133,6 +136,7 @@ static void __latent_entropy irq_poll_softirq(struct softirq_action *h)
|
2018-08-27 14:32:32 +00:00
|
|
|
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
|
|
|
|
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -196,6 +200,7 @@ static int irq_poll_cpu_dead(unsigned int cpu)
|
2018-08-27 14:32:32 +00:00
|
|
|
this_cpu_ptr(&blk_cpu_iopoll));
|
|
|
|
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/net/core/dev.c b/net/core/dev.c
|
2020-04-21 20:13:52 +00:00
|
|
|
index 2f4d35101f4d..626743ea0d53 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/net/core/dev.c
|
|
|
|
+++ b/net/core/dev.c
|
2020-01-19 00:54:59 +00:00
|
|
|
@@ -2727,6 +2727,7 @@ static void __netif_reschedule(struct Qdisc *q)
|
2018-08-27 14:32:32 +00:00
|
|
|
sd->output_queue_tailp = &q->next_sched;
|
|
|
|
raise_softirq_irqoff(NET_TX_SOFTIRQ);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
|
|
|
|
void __netif_schedule(struct Qdisc *q)
|
2020-01-19 00:54:59 +00:00
|
|
|
@@ -2789,6 +2790,7 @@ void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
|
2018-08-27 14:32:32 +00:00
|
|
|
__this_cpu_write(softnet_data.completion_queue, skb);
|
|
|
|
raise_softirq_irqoff(NET_TX_SOFTIRQ);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__dev_kfree_skb_irq);
|
|
|
|
|
2020-04-21 20:13:52 +00:00
|
|
|
@@ -4263,6 +4265,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
|
2018-08-27 14:32:32 +00:00
|
|
|
rps_unlock(sd);
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
atomic_long_inc(&skb->dev->rx_dropped);
|
|
|
|
kfree_skb(skb);
|
2020-04-21 20:13:52 +00:00
|
|
|
@@ -5803,12 +5806,14 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
|
2018-08-27 14:32:32 +00:00
|
|
|
sd->rps_ipi_list = NULL;
|
|
|
|
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
/* Send pending IPI's to kick RPS processing on remote cpus. */
|
|
|
|
net_rps_send_ipi(remsd);
|
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
|
2020-04-21 20:13:52 +00:00
|
|
|
@@ -5886,6 +5891,7 @@ void __napi_schedule(struct napi_struct *n)
|
2018-08-27 14:32:32 +00:00
|
|
|
local_irq_save(flags);
|
|
|
|
____napi_schedule(this_cpu_ptr(&softnet_data), n);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__napi_schedule);
|
|
|
|
|
2020-04-21 20:13:52 +00:00
|
|
|
@@ -9488,6 +9494,7 @@ static int dev_cpu_dead(unsigned int oldcpu)
|
2018-08-27 14:32:32 +00:00
|
|
|
|
|
|
|
raise_softirq_irqoff(NET_TX_SOFTIRQ);
|
|
|
|
local_irq_enable();
|
|
|
|
+ preempt_check_resched_rt();
|
|
|
|
|
|
|
|
#ifdef CONFIG_RPS
|
|
|
|
remsd = oldsd->rps_ipi_list;
|
2020-01-03 23:36:11 +00:00
|
|
|
--
|
2020-04-09 19:44:24 +00:00
|
|
|
2.25.1
|
2020-01-03 23:36:11 +00:00
|
|
|
|