From 1e4d3001f59fb7a9917cb746544b65e616b5f809 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 20 Nov 2023 15:33:46 +0100 Subject: x86/entry: Harden return-to-user Make the CONFIG_DEBUG_ENTRY=y check that validates CS is a user segment unconditional and move it nearer to IRET. PRE: 140,026,608 cycles:k ( +- 0.01% ) 236,696,176 instructions:k # 1.69 insn per cycle ( +- 0.00% ) POST: 139,957,681 cycles:k ( +- 0.01% ) 236,681,819 instructions:k # 1.69 insn per cycle ( +- 0.00% ) (this is with --repeat 100 and the run-to-run variance is bigger than the difference shown) Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Linus Torvalds Link: https://lore.kernel.org/r/20231120143626.753200755@infradead.org --- arch/x86/entry/entry_64.S | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'arch/x86/entry/entry_64.S') diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index dfbf799b7311..c40f89ab1b4c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -559,13 +559,6 @@ __irqentry_text_end: SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) IBRS_EXIT -#ifdef CONFIG_DEBUG_ENTRY - /* Assert that pt_regs indicates user mode. */ - testb $3, CS(%rsp) - jnz 1f - ud2 -1: -#endif #ifdef CONFIG_XEN_PV ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV #endif @@ -576,8 +569,14 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) STACKLEAK_ERASE POP_REGS add $8, %rsp /* orig_ax */ + UNWIND_HINT_IRET_REGS + +.Lswapgs_and_iret: swapgs - jmp .Lnative_iret + /* Assert that the IRET frame indicates user mode. */ + testb $3, 8(%rsp) + jnz .Lnative_iret + ud2 #ifdef CONFIG_PAGE_TABLE_ISOLATION .Lpti_restore_regs_and_return_to_usermode: @@ -613,8 +612,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi - swapgs - jmp .Lnative_iret + jmp .Lswapgs_and_iret #endif SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) -- cgit