forked from torvalds/linux
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide the corresponding thunks. Provide assembler macros for invoking the thunks in the same way that GCC does, from native and inline assembler. This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In some circumstances, IBRS microcode features may be used instead, and the retpoline can be disabled. On AMD CPUs if lfence is serialising, the retpoline can be dramatically simplified to a simple "lfence; jmp *\reg". A future patch, after it has been verified that lfence really is serialising in all circumstances, can enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition to X86_FEATURE_RETPOLINE. Do not align the retpoline in the altinstr section, because there is no guarantee that it stays aligned when it's copied over the oldinstr during alternative patching. [ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks] [ tglx: Put actual function CALL/JMP in front of the macros, convert to symbolic labels ] [ dwmw2: Convert back to numeric labels, merge objtool fixes ] Signed-off-by: David Woodhouse <[email protected]> Signed-off-by: Thomas Gleixner <[email protected]> Acked-by: Arjan van de Ven <[email protected]> Acked-by: Ingo Molnar <[email protected]> Cc: [email protected] Cc: Rik van Riel <[email protected]> Cc: Andi Kleen <[email protected]> Cc: Josh Poimboeuf <[email protected]> Cc: [email protected] Cc: Peter Zijlstra <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Jiri Kosina <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: Dave Hansen <[email protected]> Cc: Kees Cook <[email protected]> Cc: Tim Chen <[email protected]> Cc: Greg Kroah-Hartman <[email protected]> Cc: Paul Turner <[email protected]> Link: https://lkml.kernel.org/r/[email protected]
- Loading branch information
Showing
8 changed files
with
231 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| /* SPDX-License-Identifier: GPL-2.0 */ | ||
|
|
||
| #ifndef __NOSPEC_BRANCH_H__ | ||
| #define __NOSPEC_BRANCH_H__ | ||
|
|
||
| #include <asm/alternative.h> | ||
| #include <asm/alternative-asm.h> | ||
| #include <asm/cpufeatures.h> | ||
|
|
||
| #ifdef __ASSEMBLY__ | ||
|
|
||
| /* | ||
| * This should be used immediately before a retpoline alternative. It tells | ||
| * objtool where the retpolines are so that it can make sense of the control | ||
| * flow by just reading the original instruction(s) and ignoring the | ||
| * alternatives. | ||
| */ | ||
| .macro ANNOTATE_NOSPEC_ALTERNATIVE | ||
| .Lannotate_\@: | ||
| .pushsection .discard.nospec | ||
| .long .Lannotate_\@ - . | ||
| .popsection | ||
| .endm | ||
|
|
||
| /* | ||
| * These are the bare retpoline primitives for indirect jmp and call. | ||
| * Do not use these directly; they only exist to make the ALTERNATIVE | ||
| * invocation below less ugly. | ||
| */ | ||
| .macro RETPOLINE_JMP reg:req | ||
| call .Ldo_rop_\@ | ||
| .Lspec_trap_\@: | ||
| pause | ||
| jmp .Lspec_trap_\@ | ||
| .Ldo_rop_\@: | ||
| mov \reg, (%_ASM_SP) | ||
| ret | ||
| .endm | ||
|
|
||
| /* | ||
| * This is a wrapper around RETPOLINE_JMP so the called function in reg | ||
| * returns to the instruction after the macro. | ||
| */ | ||
| .macro RETPOLINE_CALL reg:req | ||
| jmp .Ldo_call_\@ | ||
| .Ldo_retpoline_jmp_\@: | ||
| RETPOLINE_JMP \reg | ||
| .Ldo_call_\@: | ||
| call .Ldo_retpoline_jmp_\@ | ||
| .endm | ||
|
|
||
| /* | ||
| * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | ||
| * indirect jmp/call which may be susceptible to the Spectre variant 2 | ||
| * attack. | ||
| */ | ||
| .macro JMP_NOSPEC reg:req | ||
| #ifdef CONFIG_RETPOLINE | ||
| ANNOTATE_NOSPEC_ALTERNATIVE | ||
| ALTERNATIVE_2 __stringify(jmp *\reg), \ | ||
| __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ | ||
| __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD | ||
| #else | ||
| jmp *\reg | ||
| #endif | ||
| .endm | ||
|
|
||
| .macro CALL_NOSPEC reg:req | ||
| #ifdef CONFIG_RETPOLINE | ||
| ANNOTATE_NOSPEC_ALTERNATIVE | ||
| ALTERNATIVE_2 __stringify(call *\reg), \ | ||
| __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ | ||
| __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD | ||
| #else | ||
| call *\reg | ||
| #endif | ||
| .endm | ||
|
|
||
| #else /* __ASSEMBLY__ */ | ||
|
|
||
| #define ANNOTATE_NOSPEC_ALTERNATIVE \ | ||
| "999:\n\t" \ | ||
| ".pushsection .discard.nospec\n\t" \ | ||
| ".long 999b - .\n\t" \ | ||
| ".popsection\n\t" | ||
|
|
||
| #if defined(CONFIG_X86_64) && defined(RETPOLINE) | ||
|
|
||
| /* | ||
| * Since the inline asm uses the %V modifier which is only in newer GCC, | ||
| * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. | ||
| */ | ||
| # define CALL_NOSPEC \ | ||
| ANNOTATE_NOSPEC_ALTERNATIVE \ | ||
| ALTERNATIVE( \ | ||
| "call *%[thunk_target]\n", \ | ||
| "call __x86_indirect_thunk_%V[thunk_target]\n", \ | ||
| X86_FEATURE_RETPOLINE) | ||
| # define THUNK_TARGET(addr) [thunk_target] "r" (addr) | ||
|
|
||
| #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) | ||
| /* | ||
| * For i386 we use the original ret-equivalent retpoline, because | ||
| * otherwise we'll run out of registers. We don't care about CET | ||
| * here, anyway. | ||
| */ | ||
| # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ | ||
| " jmp 904f;\n" \ | ||
| " .align 16\n" \ | ||
| "901: call 903f;\n" \ | ||
| "902: pause;\n" \ | ||
| " jmp 902b;\n" \ | ||
| " .align 16\n" \ | ||
| "903: addl $4, %%esp;\n" \ | ||
| " pushl %[thunk_target];\n" \ | ||
| " ret;\n" \ | ||
| " .align 16\n" \ | ||
| "904: call 901b;\n", \ | ||
| X86_FEATURE_RETPOLINE) | ||
|
|
||
| # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | ||
| #else /* No retpoline */ | ||
| # define CALL_NOSPEC "call *%[thunk_target]\n" | ||
| # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | ||
| #endif | ||
|
|
||
| #endif /* __ASSEMBLY__ */ | ||
| #endif /* __NOSPEC_BRANCH_H__ */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| /* SPDX-License-Identifier: GPL-2.0 */ | ||
|
|
||
| #include <linux/stringify.h> | ||
| #include <linux/linkage.h> | ||
| #include <asm/dwarf2.h> | ||
| #include <asm/cpufeatures.h> | ||
| #include <asm/alternative-asm.h> | ||
| #include <asm/export.h> | ||
| #include <asm/nospec-branch.h> | ||
|
|
||
| .macro THUNK reg | ||
| .section .text.__x86.indirect_thunk.\reg | ||
|
|
||
| ENTRY(__x86_indirect_thunk_\reg) | ||
| CFI_STARTPROC | ||
| JMP_NOSPEC %\reg | ||
| CFI_ENDPROC | ||
| ENDPROC(__x86_indirect_thunk_\reg) | ||
| .endm | ||
|
|
||
| /* | ||
| * Despite being an assembler file we can't just use .irp here | ||
| * because __KSYM_DEPS__ only uses the C preprocessor and would | ||
| * only see one instance of "__x86_indirect_thunk_\reg" rather | ||
| * than one per register with the correct names. So we do it | ||
| * the simple and nasty way... | ||
| */ | ||
| #define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg) | ||
| #define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg) | ||
|
|
||
| GENERATE_THUNK(_ASM_AX) | ||
| GENERATE_THUNK(_ASM_BX) | ||
| GENERATE_THUNK(_ASM_CX) | ||
| GENERATE_THUNK(_ASM_DX) | ||
| GENERATE_THUNK(_ASM_SI) | ||
| GENERATE_THUNK(_ASM_DI) | ||
| GENERATE_THUNK(_ASM_BP) | ||
| GENERATE_THUNK(_ASM_SP) | ||
| #ifdef CONFIG_64BIT | ||
| GENERATE_THUNK(r8) | ||
| GENERATE_THUNK(r9) | ||
| GENERATE_THUNK(r10) | ||
| GENERATE_THUNK(r11) | ||
| GENERATE_THUNK(r12) | ||
| GENERATE_THUNK(r13) | ||
| GENERATE_THUNK(r14) | ||
| GENERATE_THUNK(r15) | ||
| #endif |