[PATCH 17/24] x86/entry: Introduce struct ist_regs

[Lai Jiangshan]

From: Lai Jiangshan <laijs@xxxxxxxxxxxxxxxxx>

struct ist_regs is the upmost stack frame for IST interrupt, it
consists of struct pt_regs and other fields which will be added in
later patch.

Make vc_switch_off_ist() take ist_regs as its argument and it will switch
the whole ist_regs if needed.

Make the frame before calling paranoid_entry() and paranoid_exit() be
struct ist_regs.

This patch is prepared for converting paranoid_entry() and paranoid_exit()
into C code which will need the additinal fields to store the results in
paranoid_entry() and to use them in paranoid_exit().

The C interrupt handlers don't use struct ist_regs due to they don't need
the additional fields in the struct ist_regs, and they still use pt_regs.

No functional change intended and IST_pt_reg is still 0.

Signed-off-by: Lai Jiangshan <laijs@xxxxxxxxxxxxxxxxx>
---
 arch/x86/entry/entry_64.S        | 37 ++++++++++++++++++--------------
 arch/x86/entry/traps.c           | 16 +++++++-------
 arch/x86/include/asm/traps.h     | 10 ++++++++-
 arch/x86/kernel/asm-offsets_64.c |  2 ++
 4 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index e968074046c3..1ae10ca351f4 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -431,13 +431,14 @@ SYM_CODE_START(\asmsym)
 	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
 	call	paranoid_entry
 
-	UNWIND_HINT_REGS
+	UNWIND_HINT_REGS offset=IST_pt_regs
 
-	movq	%rsp, %rdi		/* pt_regs pointer */
+	leaq	IST_pt_regs(%rsp), %rdi	/* pt_regs pointer */
 
 	call	\cfunc
 
 	call	paranoid_exit
+	addq	$IST_pt_regs, %rsp	/* put %rsp back to pt_regs */
 	jmp	restore_regs_and_return_to_kernel
 
 	/* Switch to the regular task stack and use the noist entry point */
@@ -488,7 +489,7 @@ SYM_CODE_START(\asmsym)
 	 */
 	call	paranoid_entry
 
-	UNWIND_HINT_REGS
+	UNWIND_HINT_REGS offset=IST_pt_regs
 
 	/*
 	 * Switch off the IST stack to make it free for nested exceptions. The
@@ -496,17 +497,17 @@ SYM_CODE_START(\asmsym)
 	 * stack if it is safe to do so. If not it switches to the VC fall-back
 	 * stack.
 	 */
-	movq	%rsp, %rdi		/* pt_regs pointer */
+	movq	%rsp, %rdi		/* ist_regs pointer */
 	call	vc_switch_off_ist
 	movq	%rax, %rsp		/* Switch to new stack */
 
-	UNWIND_HINT_REGS
+	UNWIND_HINT_REGS offset=IST_pt_regs
 
-	/* Update pt_regs */
-	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
-	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
+	leaq	IST_pt_regs(%rsp), %rdi	/* pt_regs pointer */
 
-	movq	%rsp, %rdi		/* pt_regs pointer */
+	/* Update pt_regs */
+	movq	ORIG_RAX(%rdi), %rsi	/* get error code into 2nd argument*/
+	movq	$-1, ORIG_RAX(%rdi)	/* no syscall to restart */
 
 	call	kernel_\cfunc
 
@@ -516,6 +517,7 @@ SYM_CODE_START(\asmsym)
 	 * so it is definitely mapped even with PTI enabled.
 	 */
 	call	paranoid_exit
+	addq	$IST_pt_regs, %rsp	/* put %rsp back to pt_regs */
 	jmp	restore_regs_and_return_to_kernel
 
 	/* Switch to the regular task stack */
@@ -539,14 +541,15 @@ SYM_CODE_START(\asmsym)
 
 	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
 	call	paranoid_entry
-	UNWIND_HINT_REGS
+	UNWIND_HINT_REGS offset=IST_pt_regs
 
-	movq	%rsp, %rdi		/* pt_regs pointer into first argument */
-	movq	ORIG_RAX(%rsp), %rsi	/* get error code into 2nd argument*/
-	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
+	leaq	IST_pt_regs(%rsp), %rdi	/* pt_regs pointer into first argument */
+	movq	ORIG_RAX(%rdi), %rsi	/* get error code into 2nd argument*/
+	movq	$-1, ORIG_RAX(%rdi)	/* no syscall to restart */
 	call	\cfunc
 
 	call	paranoid_exit
+	addq	$IST_pt_regs, %rsp	/* put %rsp back to pt_regs */
 	jmp	restore_regs_and_return_to_kernel
 
 _ASM_NOKPROBE(\asmsym)
@@ -852,8 +855,9 @@ SYM_CODE_START_LOCAL(paranoid_entry)
 	PUSH_AND_CLEAR_REGS skip_rdi=1
 	movq	RDI(%rsp), %rsi	/* temporarily store the return address in %rsi */
 	movq	%rdi, RDI(%rsp) /* put %rdi onto pt_regs */
+	subq	$IST_pt_regs, %rsp /* reserve room for ist_regs */
 	pushq	%rsi		/* put the return address onto the stack */
-	ENCODE_FRAME_POINTER 8
+	ENCODE_FRAME_POINTER 8+IST_pt_regs
 
 	/*
 	 * Always stash CR3 in %r14.  This value will be restored,
@@ -1294,9 +1298,9 @@ end_repeat_nmi:
 	 * Use paranoid_entry to handle SWAPGS and CR3.
 	 */
 	call	paranoid_entry
-	UNWIND_HINT_REGS
+	UNWIND_HINT_REGS offset=IST_pt_regs
 
-	movq	%rsp, %rdi
+	leaq	IST_pt_regs(%rsp), %rdi		/* pt_regs pointer */
 	movq	$-1, %rsi
 	call	exc_nmi
 
@@ -1305,6 +1309,7 @@ end_repeat_nmi:
 	 * restore_regs_and_return_to_kernel as we must handle nested NMI.
 	 */
 	call	paranoid_exit
+	addq	$IST_pt_regs, %rsp		/* put %rsp back to pt_regs */
 
 	POP_REGS
 
diff --git a/arch/x86/entry/traps.c b/arch/x86/entry/traps.c
index 40722a2f61ae..da55565d43ef 100644
--- a/arch/x86/entry/traps.c
+++ b/arch/x86/entry/traps.c
@@ -693,17 +693,17 @@ struct pt_regs *sync_regs(struct pt_regs *eregs)
 }
 
 #ifdef CONFIG_AMD_MEM_ENCRYPT
-asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)
+asmlinkage __visible noinstr struct ist_regs *vc_switch_off_ist(struct ist_regs *ist)
 {
 	unsigned long sp, *stack;
 	struct stack_info info;
-	struct pt_regs *regs_ret;
+	struct ist_regs *ist_ret;
 
 	/*
 	 * In the SYSCALL entry path the RSP value comes from user-space - don't
 	 * trust it and switch to the current kernel stack
 	 */
-	if (ip_within_syscall_gap(regs)) {
+	if (ip_within_syscall_gap(&ist->regs)) {
 		sp = this_cpu_read(cpu_current_top_of_stack);
 		goto sync;
 	}
@@ -713,7 +713,7 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *r
 	 * happened from a safe stack. Not safe are the entry or unknown stacks,
 	 * use the fall-back stack instead in this case.
 	 */
-	sp    = regs->sp;
+	sp    = ist->regs.sp;
 	stack = (unsigned long *)sp;
 
 	if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY ||
@@ -726,12 +726,12 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *r
 	 * IST stack. The code below only copies pt_regs, the real switch happens
 	 * in assembly code.
 	 */
-	sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret);
+	sp = ALIGN_DOWN(sp, 8) - sizeof(*ist_ret);
 
-	regs_ret = (struct pt_regs *)sp;
-	*regs_ret = *regs;
+	ist_ret = (struct ist_regs *)sp;
+	*ist_ret = *ist;
 
-	return regs_ret;
+	return ist_ret;
 }
 #endif
 
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 5c41a279c1e0..e24c63bbc30a 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -11,9 +11,17 @@
 #include <asm/trap_pf.h>
 
 #ifdef CONFIG_X86_64
+struct ist_regs {
+	/*
+	 * ist specific fields must be defined before pt_regs
+	 * and they are located below pt_regs on the stacks.
+	 */
+	struct pt_regs regs;
+};
+
 asmlinkage __visible notrace struct pt_regs *do_error_entry(struct pt_regs *eregs);
 void __init trap_init(void);
-asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);
+asmlinkage __visible noinstr struct ist_regs *vc_switch_off_ist(struct ist_regs *ist);
 #endif
 
 #ifdef CONFIG_X86_F00F_BUG
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index b14533af7676..3cffadf64ac2 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -4,6 +4,7 @@
 #endif
 
 #include <asm/ia32.h>
+#include <asm/traps.h>
 
 #if defined(CONFIG_KVM_GUEST) && defined(CONFIG_PARAVIRT_SPINLOCKS)
 #include <asm/kvm_para.h>
@@ -55,6 +56,7 @@ int main(void)
 #undef ENTRY
 
 	BLANK();
+	DEFINE(IST_pt_regs, offsetof(struct ist_regs, regs));
 
 #ifdef CONFIG_STACKPROTECTOR
 	DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary));
-- 
2.19.1.6.gb485710b