105 lines
3.6 KiB
Diff
105 lines
3.6 KiB
Diff
From: Paul Burton <paulburton@kernel.org>
|
|
Date: Fri, 18 Oct 2019 15:38:48 -0700
|
|
Subject: MIPS: tlbex: Fix build_restore_pagemask KScratch restore
|
|
Origin: https://git.kernel.org/linus/b42aa3fd5957e4daf4b69129e5ce752a2a53e7d6
|
|
|
|
build_restore_pagemask() will restore the value of register $1/$at when
|
|
its restore_scratch argument is non-zero, and aims to do so by filling a
|
|
branch delay slot. Commit 0b24cae4d535 ("MIPS: Add missing EHB in mtc0
|
|
-> mfc0 sequence.") added an EHB instruction (Execution Hazard Barrier)
|
|
prior to restoring $1 from a KScratch register, in order to resolve a
|
|
hazard that can result in stale values of the KScratch register being
|
|
observed. In particular, P-class CPUs from MIPS with out of order
|
|
execution pipelines such as the P5600 & P6600 are affected.
|
|
|
|
Unfortunately this EHB instruction was inserted in the branch delay slot
|
|
causing the MFC0 instruction which performs the restoration to no longer
|
|
execute along with the branch. The result is that the $1 register isn't
|
|
actually restored, ie. the TLB refill exception handler clobbers it -
|
|
which is exactly the problem the EHB is meant to avoid for the P-class
|
|
CPUs.
|
|
|
|
Similarly build_get_pgd_vmalloc() will restore the value of $1/$at when
|
|
its mode argument equals refill_scratch, and suffers from the same
|
|
problem.
|
|
|
|
Fix this by in both cases moving the EHB earlier in the emitted code.
|
|
There's no reason it needs to immediately precede the MFC0 - it simply
|
|
needs to be between the MTC0 & MFC0.
|
|
|
|
This bug only affects Cavium Octeon systems which use
|
|
build_fast_tlb_refill_handler().
|
|
|
|
Signed-off-by: Paul Burton <paulburton@kernel.org>
|
|
Fixes: 0b24cae4d535 ("MIPS: Add missing EHB in mtc0 -> mfc0 sequence.")
|
|
Cc: Dmitry Korotin <dkorotin@wavecomp.com>
|
|
Cc: stable@vger.kernel.org # v3.15+
|
|
Cc: linux-mips@vger.kernel.org
|
|
Cc: linux-kernel@vger.kernel.org
|
|
---
|
|
arch/mips/mm/tlbex.c | 23 +++++++++++++++--------
|
|
1 file changed, 15 insertions(+), 8 deletions(-)
|
|
|
|
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
|
|
index e01cb33bfa1a..41bb91f05688 100644
|
|
--- a/arch/mips/mm/tlbex.c
|
|
+++ b/arch/mips/mm/tlbex.c
|
|
@@ -653,6 +653,13 @@ static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
|
|
int restore_scratch)
|
|
{
|
|
if (restore_scratch) {
|
|
+ /*
|
|
+ * Ensure the MFC0 below observes the value written to the
|
|
+ * KScratch register by the prior MTC0.
|
|
+ */
|
|
+ if (scratch_reg >= 0)
|
|
+ uasm_i_ehb(p);
|
|
+
|
|
/* Reset default page size */
|
|
if (PM_DEFAULT_MASK >> 16) {
|
|
uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
@@ -667,12 +674,10 @@ static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
|
|
uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
uasm_il_b(p, r, lid);
|
|
}
|
|
- if (scratch_reg >= 0) {
|
|
- uasm_i_ehb(p);
|
|
+ if (scratch_reg >= 0)
|
|
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
|
|
- } else {
|
|
+ else
|
|
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
|
- }
|
|
} else {
|
|
/* Reset default page size */
|
|
if (PM_DEFAULT_MASK >> 16) {
|
|
@@ -921,6 +926,10 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
}
|
|
if (mode != not_refill && check_for_high_segbits) {
|
|
uasm_l_large_segbits_fault(l, *p);
|
|
+
|
|
+ if (mode == refill_scratch && scratch_reg >= 0)
|
|
+ uasm_i_ehb(p);
|
|
+
|
|
/*
|
|
* We get here if we are an xsseg address, or if we are
|
|
* an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
|
|
@@ -939,12 +948,10 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
uasm_i_jr(p, ptr);
|
|
|
|
if (mode == refill_scratch) {
|
|
- if (scratch_reg >= 0) {
|
|
- uasm_i_ehb(p);
|
|
+ if (scratch_reg >= 0)
|
|
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
|
|
- } else {
|
|
+ else
|
|
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
|
- }
|
|
} else {
|
|
uasm_i_nop(p);
|
|
}
|
|
--
|
|
2.24.0
|
|
|