From: Peter Zijlstra peterz@infradead.org
[ Upstream commit aa93e2ad7464ffb90155a5ffdde963816f86d5dc ]
Where possible, push the .fixup into code, at the tail of functions.
This is hard for macros since they're used in multiple functions, therefore introduce a new extable handler to pop zeros.
Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Reviewed-by: Josh Poimboeuf jpoimboe@redhat.com Link: https://lore.kernel.org/r/20211110101325.245184699@infradead.org Signed-off-by: Sasha Levin sashal@kernel.org --- arch/x86/entry/entry_32.S | 28 +++++++--------------- arch/x86/include/asm/extable_fixup_types.h | 2 ++ arch/x86/mm/extable.c | 14 +++++++++++ 3 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 5bd3baf36d87..2cba70f9753b 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -270,17 +270,9 @@ 3: popl %fs addl $(4 + \pop), %esp /* pop the unused "gs" slot */ IRET_FRAME -.pushsection .fixup, "ax" -4: movl $0, (%esp) - jmp 1b -5: movl $0, (%esp) - jmp 2b -6: movl $0, (%esp) - jmp 3b -.popsection - _ASM_EXTABLE(1b, 4b) - _ASM_EXTABLE(2b, 5b) - _ASM_EXTABLE(3b, 6b) + _ASM_EXTABLE_TYPE(1b, 1b, EX_TYPE_POP_ZERO) + _ASM_EXTABLE_TYPE(2b, 2b, EX_TYPE_POP_ZERO) + _ASM_EXTABLE_TYPE(3b, 3b, EX_TYPE_POP_ZERO) .endm
.macro RESTORE_ALL_NMI cr3_reg:req pop=0 @@ -923,10 +915,8 @@ SYM_FUNC_START(entry_SYSENTER_32) sti sysexit
-.pushsection .fixup, "ax" -2: movl $0, PT_FS(%esp) - jmp 1b -.popsection +2: movl $0, PT_FS(%esp) + jmp 1b _ASM_EXTABLE(1b, 2b)
.Lsysenter_fix_flags: @@ -994,8 +984,7 @@ restore_all_switch_stack: */ iret
-.section .fixup, "ax" -SYM_CODE_START(asm_iret_error) +.Lasm_iret_error: pushl $0 # no error code pushl $iret_error
@@ -1012,9 +1001,8 @@ SYM_CODE_START(asm_iret_error) #endif
jmp handle_exception -SYM_CODE_END(asm_iret_error) -.previous - _ASM_EXTABLE(.Lirq_return, asm_iret_error) + + _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error) SYM_FUNC_END(entry_INT80_32)
.macro FIXUP_ESPFIX_STACK diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h index 409524d5d2eb..4d709a2768bb 100644 --- a/arch/x86/include/asm/extable_fixup_types.h +++ b/arch/x86/include/asm/extable_fixup_types.h @@ -19,4 +19,6 @@ #define EX_TYPE_DEFAULT_MCE_SAFE 12 #define EX_TYPE_FAULT_MCE_SAFE 13
+#define EX_TYPE_POP_ZERO 14 + #endif diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index f37e290e6d0a..f59a4d017070 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -99,6 +99,18 @@ static bool ex_handler_clear_fs(const struct exception_table_entry *fixup, return ex_handler_default(fixup, regs); }
+static bool ex_handler_pop_zero(const struct exception_table_entry *fixup, + struct pt_regs *regs) +{ + /* + * Typically used for when "pop %seg" traps, in which case we'll clear + * the stack slot and re-try the instruction, which will then succeed + * to pop zero. + */ + *((unsigned long *)regs->sp) = 0; + return ex_handler_default(fixup, regs); +} + int ex_get_fixup_type(unsigned long ip) { const struct exception_table_entry *e = search_exception_tables(ip); @@ -156,6 +168,8 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, case EX_TYPE_WRMSR_IN_MCE: ex_handler_msr_mce(regs, true); break; + case EX_TYPE_POP_ZERO: + return ex_handler_pop_zero(e, regs); } BUG(); }