2020-09-04 20:10:21 +00:00
|
|
|
From d191b91aa1860dae76957c01906d1c1f92191f08 Mon Sep 17 00:00:00 2001
|
|
|
|
Message-Id: <d191b91aa1860dae76957c01906d1c1f92191f08.1599166691.git.zanussi@kernel.org>
|
|
|
|
In-Reply-To: <56457dc415803c8abc5acb513ada877a79596f05.1599166690.git.zanussi@kernel.org>
|
|
|
|
References: <56457dc415803c8abc5acb513ada877a79596f05.1599166690.git.zanussi@kernel.org>
|
2018-08-27 14:32:32 +00:00
|
|
|
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
|
|
Date: Thu, 30 Nov 2017 13:40:10 +0100
|
2020-09-04 20:10:21 +00:00
|
|
|
Subject: [PATCH 222/333] crypto: limit more FPU-enabled sections
|
2018-08-27 14:32:32 +00:00
|
|
|
MIME-Version: 1.0
|
|
|
|
Content-Type: text/plain; charset=UTF-8
|
|
|
|
Content-Transfer-Encoding: 8bit
|
2020-09-04 20:10:21 +00:00
|
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.142-rt63.tar.xz
|
2018-08-27 14:32:32 +00:00
|
|
|
|
|
|
|
Those crypto drivers use SSE/AVX/… for their crypto work and in order to
|
|
|
|
do so in kernel they need to enable the "FPU" in kernel mode which
|
|
|
|
disables preemption.
|
|
|
|
There are two problems with the way they are used:
|
|
|
|
- the while loop which processes X bytes may create latency spikes and
|
|
|
|
should be avoided or limited.
|
|
|
|
- the cipher-walk-next part may allocate/free memory and may use
|
|
|
|
kmap_atomic().
|
|
|
|
|
|
|
|
The whole kernel_fpu_begin()/end() processing isn't probably that cheap.
|
|
|
|
It most likely makes sense to process as much of those as possible in one
|
|
|
|
go. The new *_fpu_sched_rt() schedules only if a RT task is pending.
|
|
|
|
|
|
|
|
Probably we should measure the performance those ciphers in pure SW
|
|
|
|
mode and with this optimisations to see if it makes sense to keep them
|
|
|
|
for RT.
|
|
|
|
|
|
|
|
This kernel_fpu_resched() makes the code more preemptible which might hurt
|
|
|
|
performance.
|
|
|
|
|
|
|
|
Cc: stable-rt@vger.kernel.org
|
|
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
|
|
---
|
2019-04-08 23:49:20 +00:00
|
|
|
arch/x86/crypto/chacha20_glue.c | 9 +++++----
|
|
|
|
arch/x86/include/asm/fpu/api.h | 1 +
|
|
|
|
arch/x86/kernel/fpu/core.c | 12 ++++++++++++
|
2018-08-27 14:32:32 +00:00
|
|
|
3 files changed, 18 insertions(+), 4 deletions(-)
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c
|
|
|
|
index dce7c5d39c2f..6194160b7fbc 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/arch/x86/crypto/chacha20_glue.c
|
|
|
|
+++ b/arch/x86/crypto/chacha20_glue.c
|
2019-04-08 23:49:20 +00:00
|
|
|
@@ -81,23 +81,24 @@ static int chacha20_simd(struct skcipher_request *req)
|
2018-08-27 14:32:32 +00:00
|
|
|
|
|
|
|
crypto_chacha20_init(state, ctx, walk.iv);
|
|
|
|
|
|
|
|
- kernel_fpu_begin();
|
|
|
|
-
|
|
|
|
while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
|
|
|
|
+ kernel_fpu_begin();
|
|
|
|
+
|
|
|
|
chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
|
|
|
|
rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
|
|
|
|
+ kernel_fpu_end();
|
|
|
|
err = skcipher_walk_done(&walk,
|
|
|
|
walk.nbytes % CHACHA20_BLOCK_SIZE);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (walk.nbytes) {
|
|
|
|
+ kernel_fpu_begin();
|
|
|
|
chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
|
|
|
|
walk.nbytes);
|
|
|
|
+ kernel_fpu_end();
|
|
|
|
err = skcipher_walk_done(&walk, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
- kernel_fpu_end();
|
|
|
|
-
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h
|
2019-08-18 21:33:52 +00:00
|
|
|
index b56d504af654..e51c7094075d 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/arch/x86/include/asm/fpu/api.h
|
|
|
|
+++ b/arch/x86/include/asm/fpu/api.h
|
2019-08-18 21:33:52 +00:00
|
|
|
@@ -20,6 +20,7 @@
|
|
|
|
*/
|
2018-08-27 14:32:32 +00:00
|
|
|
extern void kernel_fpu_begin(void);
|
|
|
|
extern void kernel_fpu_end(void);
|
|
|
|
+extern void kernel_fpu_resched(void);
|
|
|
|
extern bool irq_fpu_usable(void);
|
|
|
|
|
|
|
|
/*
|
2019-04-08 23:49:20 +00:00
|
|
|
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
|
2019-08-18 21:33:52 +00:00
|
|
|
index 2e5003fef51a..768c53767bb2 100644
|
2018-08-27 14:32:32 +00:00
|
|
|
--- a/arch/x86/kernel/fpu/core.c
|
|
|
|
+++ b/arch/x86/kernel/fpu/core.c
|
2019-08-18 21:33:52 +00:00
|
|
|
@@ -136,6 +136,18 @@ void kernel_fpu_end(void)
|
2018-08-27 14:32:32 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kernel_fpu_end);
|
|
|
|
|
|
|
|
+void kernel_fpu_resched(void)
|
|
|
|
+{
|
|
|
|
+ WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
|
|
|
|
+
|
|
|
|
+ if (should_resched(PREEMPT_OFFSET)) {
|
|
|
|
+ kernel_fpu_end();
|
|
|
|
+ cond_resched();
|
|
|
|
+ kernel_fpu_begin();
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+EXPORT_SYMBOL_GPL(kernel_fpu_resched);
|
|
|
|
+
|
|
|
|
/*
|
|
|
|
* Save the FPU state (mark it for reload if necessary):
|
|
|
|
*
|
2020-01-03 23:36:11 +00:00
|
|
|
--
|
2020-06-22 13:14:16 +00:00
|
|
|
2.17.1
|
2020-01-03 23:36:11 +00:00
|
|
|
|