294 lines
10 KiB
Diff
294 lines
10 KiB
Diff
From 41cf08fe25d006f91cd7ad1a97b7729a4f0c7a20 Mon Sep 17 00:00:00 2001
|
|
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
Date: Wed, 14 Sep 2016 17:36:35 +0200
|
|
Subject: [PATCH 201/325] net/Qdisc: use a seqlock instead seqcount
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.115-rt48.tar.xz
|
|
|
|
The seqcount disables preemption on -RT while it is held which can't
|
|
remove. Also we don't want the reader to spin for ages if the writer is
|
|
scheduled out. The seqlock on the other hand will serialize / sleep on
|
|
the lock while writer is active.
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
---
|
|
include/linux/seqlock.h | 9 +++++++++
|
|
include/net/gen_stats.h | 9 +++++----
|
|
include/net/net_seq_lock.h | 15 +++++++++++++++
|
|
include/net/sch_generic.h | 19 +++++++++++++++++--
|
|
net/core/gen_estimator.c | 6 +++---
|
|
net/core/gen_stats.c | 8 ++++----
|
|
net/sched/sch_api.c | 2 +-
|
|
net/sched/sch_generic.c | 12 ++++++++++++
|
|
8 files changed, 66 insertions(+), 14 deletions(-)
|
|
create mode 100644 include/net/net_seq_lock.h
|
|
|
|
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
|
|
index 689ed53016c7..58f9909d6659 100644
|
|
--- a/include/linux/seqlock.h
|
|
+++ b/include/linux/seqlock.h
|
|
@@ -482,6 +482,15 @@ static inline void write_seqlock(seqlock_t *sl)
|
|
__raw_write_seqcount_begin(&sl->seqcount);
|
|
}
|
|
|
|
+static inline int try_write_seqlock(seqlock_t *sl)
|
|
+{
|
|
+ if (spin_trylock(&sl->lock)) {
|
|
+ __raw_write_seqcount_begin(&sl->seqcount);
|
|
+ return 1;
|
|
+ }
|
|
+ return 0;
|
|
+}
|
|
+
|
|
static inline void write_sequnlock(seqlock_t *sl)
|
|
{
|
|
__raw_write_seqcount_end(&sl->seqcount);
|
|
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
|
|
index 883bb9085f15..3b593cdeb9af 100644
|
|
--- a/include/net/gen_stats.h
|
|
+++ b/include/net/gen_stats.h
|
|
@@ -6,6 +6,7 @@
|
|
#include <linux/socket.h>
|
|
#include <linux/rtnetlink.h>
|
|
#include <linux/pkt_sched.h>
|
|
+#include <net/net_seq_lock.h>
|
|
|
|
struct gnet_stats_basic_cpu {
|
|
struct gnet_stats_basic_packed bstats;
|
|
@@ -36,11 +37,11 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type,
|
|
spinlock_t *lock, struct gnet_dump *d,
|
|
int padattr);
|
|
|
|
-int gnet_stats_copy_basic(const seqcount_t *running,
|
|
+int gnet_stats_copy_basic(net_seqlock_t *running,
|
|
struct gnet_dump *d,
|
|
struct gnet_stats_basic_cpu __percpu *cpu,
|
|
struct gnet_stats_basic_packed *b);
|
|
-void __gnet_stats_copy_basic(const seqcount_t *running,
|
|
+void __gnet_stats_copy_basic(net_seqlock_t *running,
|
|
struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu,
|
|
struct gnet_stats_basic_packed *b);
|
|
@@ -60,13 +61,13 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
|
|
struct net_rate_estimator __rcu **rate_est,
|
|
spinlock_t *lock,
|
|
- seqcount_t *running, struct nlattr *opt);
|
|
+ net_seqlock_t *running, struct nlattr *opt);
|
|
void gen_kill_estimator(struct net_rate_estimator __rcu **ptr);
|
|
int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
|
|
struct net_rate_estimator __rcu **ptr,
|
|
spinlock_t *lock,
|
|
- seqcount_t *running, struct nlattr *opt);
|
|
+ net_seqlock_t *running, struct nlattr *opt);
|
|
bool gen_estimator_active(struct net_rate_estimator __rcu **ptr);
|
|
bool gen_estimator_read(struct net_rate_estimator __rcu **ptr,
|
|
struct gnet_stats_rate_est64 *sample);
|
|
diff --git a/include/net/net_seq_lock.h b/include/net/net_seq_lock.h
|
|
new file mode 100644
|
|
index 000000000000..a7034298a82a
|
|
--- /dev/null
|
|
+++ b/include/net/net_seq_lock.h
|
|
@@ -0,0 +1,15 @@
|
|
+#ifndef __NET_NET_SEQ_LOCK_H__
|
|
+#define __NET_NET_SEQ_LOCK_H__
|
|
+
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+# define net_seqlock_t seqlock_t
|
|
+# define net_seq_begin(__r) read_seqbegin(__r)
|
|
+# define net_seq_retry(__r, __s) read_seqretry(__r, __s)
|
|
+
|
|
+#else
|
|
+# define net_seqlock_t seqcount_t
|
|
+# define net_seq_begin(__r) read_seqcount_begin(__r)
|
|
+# define net_seq_retry(__r, __s) read_seqcount_retry(__r, __s)
|
|
+#endif
|
|
+
|
|
+#endif
|
|
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
|
|
index d737a6a2600b..2d35b952bf60 100644
|
|
--- a/include/net/sch_generic.h
|
|
+++ b/include/net/sch_generic.h
|
|
@@ -10,6 +10,7 @@
|
|
#include <linux/percpu.h>
|
|
#include <linux/dynamic_queue_limits.h>
|
|
#include <linux/list.h>
|
|
+#include <net/net_seq_lock.h>
|
|
#include <linux/refcount.h>
|
|
#include <linux/workqueue.h>
|
|
#include <net/gen_stats.h>
|
|
@@ -100,7 +101,7 @@ struct Qdisc {
|
|
struct sk_buff_head gso_skb ____cacheline_aligned_in_smp;
|
|
struct qdisc_skb_head q;
|
|
struct gnet_stats_basic_packed bstats;
|
|
- seqcount_t running;
|
|
+ net_seqlock_t running;
|
|
struct gnet_stats_queue qstats;
|
|
unsigned long state;
|
|
struct Qdisc *next_sched;
|
|
@@ -121,7 +122,11 @@ static inline bool qdisc_is_running(struct Qdisc *qdisc)
|
|
{
|
|
if (qdisc->flags & TCQ_F_NOLOCK)
|
|
return spin_is_locked(&qdisc->seqlock);
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ return spin_is_locked(&qdisc->running.lock) ? true : false;
|
|
+#else
|
|
return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
|
|
+#endif
|
|
}
|
|
|
|
static inline bool qdisc_run_begin(struct Qdisc *qdisc)
|
|
@@ -132,17 +137,27 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
|
|
} else if (qdisc_is_running(qdisc)) {
|
|
return false;
|
|
}
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ if (try_write_seqlock(&qdisc->running))
|
|
+ return true;
|
|
+ return false;
|
|
+#else
|
|
/* Variant of write_seqcount_begin() telling lockdep a trylock
|
|
* was attempted.
|
|
*/
|
|
raw_write_seqcount_begin(&qdisc->running);
|
|
seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
|
|
return true;
|
|
+#endif
|
|
}
|
|
|
|
static inline void qdisc_run_end(struct Qdisc *qdisc)
|
|
{
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ write_sequnlock(&qdisc->running);
|
|
+#else
|
|
write_seqcount_end(&qdisc->running);
|
|
+#endif
|
|
if (qdisc->flags & TCQ_F_NOLOCK)
|
|
spin_unlock(&qdisc->seqlock);
|
|
}
|
|
@@ -459,7 +474,7 @@ static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)
|
|
return qdisc_lock(root);
|
|
}
|
|
|
|
-static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
|
|
+static inline net_seqlock_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
|
|
{
|
|
struct Qdisc *root = qdisc_root_sleeping(qdisc);
|
|
|
|
diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
|
|
index e4e442d70c2d..c8fa906733fb 100644
|
|
--- a/net/core/gen_estimator.c
|
|
+++ b/net/core/gen_estimator.c
|
|
@@ -46,7 +46,7 @@
|
|
struct net_rate_estimator {
|
|
struct gnet_stats_basic_packed *bstats;
|
|
spinlock_t *stats_lock;
|
|
- seqcount_t *running;
|
|
+ net_seqlock_t *running;
|
|
struct gnet_stats_basic_cpu __percpu *cpu_bstats;
|
|
u8 ewma_log;
|
|
u8 intvl_log; /* period : (250ms << intvl_log) */
|
|
@@ -129,7 +129,7 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
|
|
struct net_rate_estimator __rcu **rate_est,
|
|
spinlock_t *lock,
|
|
- seqcount_t *running,
|
|
+ net_seqlock_t *running,
|
|
struct nlattr *opt)
|
|
{
|
|
struct gnet_estimator *parm = nla_data(opt);
|
|
@@ -227,7 +227,7 @@ int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
|
|
struct net_rate_estimator __rcu **rate_est,
|
|
spinlock_t *lock,
|
|
- seqcount_t *running, struct nlattr *opt)
|
|
+ net_seqlock_t *running, struct nlattr *opt)
|
|
{
|
|
return gen_new_estimator(bstats, cpu_bstats, rate_est,
|
|
lock, running, opt);
|
|
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
|
|
index e2fd8baec65f..8bab88738691 100644
|
|
--- a/net/core/gen_stats.c
|
|
+++ b/net/core/gen_stats.c
|
|
@@ -142,7 +142,7 @@ __gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats,
|
|
}
|
|
|
|
void
|
|
-__gnet_stats_copy_basic(const seqcount_t *running,
|
|
+__gnet_stats_copy_basic(net_seqlock_t *running,
|
|
struct gnet_stats_basic_packed *bstats,
|
|
struct gnet_stats_basic_cpu __percpu *cpu,
|
|
struct gnet_stats_basic_packed *b)
|
|
@@ -155,10 +155,10 @@ __gnet_stats_copy_basic(const seqcount_t *running,
|
|
}
|
|
do {
|
|
if (running)
|
|
- seq = read_seqcount_begin(running);
|
|
+ seq = net_seq_begin(running);
|
|
bstats->bytes = b->bytes;
|
|
bstats->packets = b->packets;
|
|
- } while (running && read_seqcount_retry(running, seq));
|
|
+ } while (running && net_seq_retry(running, seq));
|
|
}
|
|
EXPORT_SYMBOL(__gnet_stats_copy_basic);
|
|
|
|
@@ -176,7 +176,7 @@ EXPORT_SYMBOL(__gnet_stats_copy_basic);
|
|
* if the room in the socket buffer was not sufficient.
|
|
*/
|
|
int
|
|
-gnet_stats_copy_basic(const seqcount_t *running,
|
|
+gnet_stats_copy_basic(net_seqlock_t *running,
|
|
struct gnet_dump *d,
|
|
struct gnet_stats_basic_cpu __percpu *cpu,
|
|
struct gnet_stats_basic_packed *b)
|
|
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
|
|
index 39e319d04bb8..fe99928aff78 100644
|
|
--- a/net/sched/sch_api.c
|
|
+++ b/net/sched/sch_api.c
|
|
@@ -1166,7 +1166,7 @@ static struct Qdisc *qdisc_create(struct net_device *dev,
|
|
rcu_assign_pointer(sch->stab, stab);
|
|
}
|
|
if (tca[TCA_RATE]) {
|
|
- seqcount_t *running;
|
|
+ net_seqlock_t *running;
|
|
|
|
err = -EOPNOTSUPP;
|
|
if (sch->flags & TCQ_F_MQROOT) {
|
|
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
|
|
index 4ab20f1138fd..a9ed58ca3924 100644
|
|
--- a/net/sched/sch_generic.c
|
|
+++ b/net/sched/sch_generic.c
|
|
@@ -575,7 +575,11 @@ struct Qdisc noop_qdisc = {
|
|
.ops = &noop_qdisc_ops,
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
|
|
.dev_queue = &noop_netdev_queue,
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ .running = __SEQLOCK_UNLOCKED(noop_qdisc.running),
|
|
+#else
|
|
.running = SEQCNT_ZERO(noop_qdisc.running),
|
|
+#endif
|
|
.busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
|
|
.gso_skb = {
|
|
.next = (struct sk_buff *)&noop_qdisc.gso_skb,
|
|
@@ -876,9 +880,17 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
|
|
lockdep_set_class(&sch->busylock,
|
|
dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ seqlock_init(&sch->running);
|
|
+ lockdep_set_class(&sch->running.seqcount,
|
|
+ dev->qdisc_running_key ?: &qdisc_running_key);
|
|
+ lockdep_set_class(&sch->running.lock,
|
|
+ dev->qdisc_running_key ?: &qdisc_running_key);
|
|
+#else
|
|
seqcount_init(&sch->running);
|
|
lockdep_set_class(&sch->running,
|
|
dev->qdisc_running_key ?: &qdisc_running_key);
|
|
+#endif
|
|
|
|
sch->ops = ops;
|
|
sch->flags = ops->static_flags;
|
|
--
|
|
2.25.1
|
|
|