[PATCH v1 RFC Zisslpcfi 09/20] riscv mmu: riscv shadow stack page fault handling

[Deepak Gupta]

Shadow stack load/stores to valid non-shadow memory raise access faults.
Regular store to shadow stack memory raise access fault as well.

This patch implements load and store access handler. Load access handler
reads faulting instruction and if it was an instruction issuing ss load,
it'll invoke page fault handler with a synthetic cause (marked reserved
in priv spec).
Similarly store access hanlder reads faulting instruction and if it was
an instruction issuing ss store, it'll invoke page fault handler with a
synthetic cause (reserved in spec).

All other cases in load/store access handler will lead to SIGSEV.

There might be concerns that using a reserved exception code may create
an issue because some riscv implementation might already using this code.
However counter argument would be, linux kernel is not using this code
and thus linux kernel should be able to use this exception code on such
a hardware.

Signed-off-by: Deepak Gupta <debug@xxxxxxxxxxxx>
---
 arch/riscv/include/asm/csr.h |  3 ++
 arch/riscv/kernel/traps.c    | 99 ++++++++++++++++++++++++++++++++++++
 arch/riscv/mm/fault.c        | 23 ++++++++-
 3 files changed, 124 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 243031d1d305..828b1c2a74c2 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -104,6 +104,9 @@
 #define EXC_SUPERVISOR_SYSCALL	10
 #define EXC_INST_PAGE_FAULT	12
 #define EXC_LOAD_PAGE_FAULT	13
+#ifdef CONFIG_USER_SHADOW_STACK
+#define EXC_SS_ACCESS_PAGE_FAULT	14
+#endif
 #define EXC_STORE_PAGE_FAULT	15
 #define EXC_INST_GUEST_PAGE_FAULT	20
 #define EXC_LOAD_GUEST_PAGE_FAULT	21
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 549bde5c970a..5553b8d48ba5 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -94,6 +94,85 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code,
 	}
 }
 
+/* Zisslpcfi instructions encodings */
+#define SS_PUSH_POP 0x81C04073
+#define SS_AMOSWAP  0x82004073
+
+bool is_ss_load_store_insn(unsigned long insn)
+{
+	if ((insn & SS_PUSH_POP) == SS_PUSH_POP)
+		return true;
+	/*
+	 * SS_AMOSWAP overlaps with LP_S_LL.
+	 * But LP_S_LL can never raise access fault
+	 */
+	if ((insn & SS_AMOSWAP) == SS_AMOSWAP)
+		return true;
+
+	return false;
+}
+
+ulong get_instruction(ulong epc)
+{
+	ulong *epc_ptr = (ulong *) epc;
+	ulong insn = 0;
+
+	__enable_user_access();
+	insn = *epc_ptr;
+	__disable_user_access();
+	return insn;
+}
+
+#ifdef CONFIG_USER_SHADOW_STACK
+extern asmlinkage void do_page_fault(struct pt_regs *regs);
+
+/*
+ * If CFI enabled then following then load access fault can occur if
+ * ssload (sspop/ssamoswap) happens on non-shadow stack memory.
+ * This is a valid case when we want to do COW on SS memory on `fork` or memory is swapped out.
+ * SS memory is marked as readonly and subsequent sspop or sspush will lead to
+ * load/store access fault. We need to decode instruction. If it's sspop or sspush
+ * Page fault handler is invoked.
+ */
+int handle_load_access_fault(struct pt_regs *regs)
+{
+	ulong insn = get_instruction(regs->epc);
+
+	if (is_ss_load_store_insn(insn)) {
+		regs->cause = EXC_SS_ACCESS_PAGE_FAULT;
+		do_page_fault(regs);
+		return 0;
+	}
+
+	return 1;
+}
+/*
+ * If CFI enabled then following then store access fault can occur if
+ * -- ssstore (sspush/ssamoswap) happens on non-shadow stack memory
+ * -- regular store happens on shadow stack memory
+ */
+int handle_store_access_fault(struct pt_regs *regs)
+{
+	ulong insn = get_instruction(regs->epc);
+
+	/*
+	* if a shadow stack store insn, change cause to
+	* synthetic SS_ACCESS_PAGE_FAULT
+	*/
+	if (is_ss_load_store_insn(insn)) {
+	        regs->cause = EXC_SS_ACCESS_PAGE_FAULT;
+	        do_page_fault(regs);
+	        return 0;
+	}
+	/*
+	 * Reaching here means it was a regular store.
+	 * A regular access fault anyways had been delivering SIGSEV
+	 * A regular store to shadow stack anyways is also a SIGSEV
+	 */
+	return 1;
+}
+#endif
+
 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
 #define __trap_section		__section(".xip.traps")
 #else
@@ -113,8 +192,18 @@ DO_ERROR_INFO(do_trap_insn_fault,
 	SIGSEGV, SEGV_ACCERR, "instruction access fault");
 DO_ERROR_INFO(do_trap_insn_illegal,
 	SIGILL, ILL_ILLOPC, "illegal instruction");
+#ifdef CONFIG_USER_SHADOW_STACK
+asmlinkage void __trap_section do_trap_load_fault(struct pt_regs *regs)
+{
+	if (!handle_load_access_fault(regs))
+		return;
+	do_trap_error(regs, SIGSEGV, SEGV_ACCERR, regs->epc,
+		"load access fault");
+}
+#else
 DO_ERROR_INFO(do_trap_load_fault,
 	SIGSEGV, SEGV_ACCERR, "load access fault");
+#endif
 #ifndef CONFIG_RISCV_M_MODE
 DO_ERROR_INFO(do_trap_load_misaligned,
 	SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
@@ -140,8 +229,18 @@ asmlinkage void __trap_section do_trap_store_misaligned(struct pt_regs *regs)
 		      "Oops - store (or AMO) address misaligned");
 }
 #endif
+#ifdef CONFIG_USER_SHADOW_STACK
+asmlinkage void __trap_section do_trap_store_fault(struct pt_regs *regs)
+{
+	if (!handle_store_access_fault(regs))
+		return;
+	do_trap_error(regs, SIGSEGV, SEGV_ACCERR, regs->epc,
+		"store (or AMO) access fault");
+}
+#else
 DO_ERROR_INFO(do_trap_store_fault,
 	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
+#endif
 DO_ERROR_INFO(do_trap_ecall_u,
 	SIGILL, ILL_ILLTRP, "environment call from U-mode");
 DO_ERROR_INFO(do_trap_ecall_s,
diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c
index d86f7cebd4a7..b5ecf36eba3d 100644
--- a/arch/riscv/mm/fault.c
+++ b/arch/riscv/mm/fault.c
@@ -18,6 +18,7 @@
 
 #include <asm/ptrace.h>
 #include <asm/tlbflush.h>
+#include <asm/pgtable.h>
 
 #include "../kernel/head.h"
 
@@ -177,6 +178,7 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a
 
 static inline bool access_error(unsigned long cause, struct vm_area_struct *vma)
 {
+	unsigned long prot = 0, shdw_stk_mask = 0;
 	switch (cause) {
 	case EXC_INST_PAGE_FAULT:
 		if (!(vma->vm_flags & VM_EXEC)) {
@@ -194,6 +196,20 @@ static inline bool access_error(unsigned long cause, struct vm_area_struct *vma)
 			return true;
 		}
 		break;
+#ifdef CONFIG_USER_SHADOW_STACK
+	/*
+	 * If a ss access page fault. vma must have only VM_WRITE.
+	 * and page prot much match to PAGE_SHADOWSTACK.
+	 */
+	case EXC_SS_ACCESS_PAGE_FAULT:
+		prot = pgprot_val(vma->vm_page_prot);
+		shdw_stk_mask = pgprot_val(PAGE_SHADOWSTACK);
+		if (((vma->vm_flags & (VM_WRITE | VM_READ | VM_EXEC)) != VM_WRITE) ||
+		   ((prot & shdw_stk_mask) != shdw_stk_mask)) {
+			return true;
+		}
+		break;
+#endif
 	default:
 		panic("%s: unhandled cause %lu", __func__, cause);
 	}
@@ -274,7 +290,12 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
 
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
 
-	if (cause == EXC_STORE_PAGE_FAULT)
+	if (cause == EXC_STORE_PAGE_FAULT
+#ifdef CONFIG_USER_SHADOW_STACK
+	   || cause == EXC_SS_ACCESS_PAGE_FAULT
+	   /* if config says shadow stack and cause is ss access then indicate a write */
+#endif
+	   )
 		flags |= FAULT_FLAG_WRITE;
 	else if (cause == EXC_INST_PAGE_FAULT)
 		flags |= FAULT_FLAG_INSTRUCTION;
-- 
2.25.1