s390/uaccess: Initialize code pages executed with non-default access key

cmpxchg_user_key() may be executed with a non-zero key; if then the storage
key of the page which belongs to the cmpxchg_user_key() code contains a key
with fetch-protection enabled the result is a protection exception:

Unable to handle kernel pointer dereference in virtual kernel address space
Failing address: 0000000000000000 TEID: 000000000000080b
Fault in home space mode while using kernel ASCE.
AS:0000000002528007 R3:00000001ffffc007 S:00000001ffffb801 P:000000000000013d
Oops: 0004 ilc:1 [#1]SMP
Modules linked in:
CPU: 3 UID: 0 PID: 791 Comm: memop Not tainted 6.16.0-rc1-00006-g3b568201d0a6-dirty #11 NONE
Hardware name: IBM 3931 A01 704 (z/VM 7.4.0)
Krnl PSW : 0794f00180000000 000003ffe0f4d91e (__cmpxchg_user_key1+0xbe/0x190)
           R:0 T:1 IO:1 EX:1 Key:9 M:1 W:0 P:0 AS:3 CC:3 PM:0 RI:0 EA:3
Krnl GPRS: 070003ffdfbf6af0 0000000000070000 0000000095b5a300 0000000000000000
           00000000f1000000 0000000000000000 0000000000000090 0000000000000000
           0000000000000040 0000000000000018 000003ff9b23d000 0000037fe0ef7bd8
           000003ffdfbf7500 00000000962e4000 0000037f00ffffff 0000037fe0ef7aa0
Krnl Code: 000003ffe0f4d912: ad03f0a0            stosm   160(%r15),3
           000003ffe0f4d916: a7780000            lhi     %r7,0
          #000003ffe0f4d91a: b20a6000            spka    0(%r6)
          >000003ffe0f4d91e: b2790100            sacf    256
           000003ffe0f4d922: a56f0080            llill   %r6,128
           000003ffe0f4d926: 5810a000            l       %r1,0(%r10)
           000003ffe0f4d92a: 141e                nr      %r1,%r14
           000003ffe0f4d92c: c0e7ffffffff        xilf    %r14,4294967295
Call Trace:
 [<000003ffe0f4d91e>] __cmpxchg_user_key1+0xbe/0x190
 [<000003ffe0189c6e>] cmpxchg_guest_abs_with_key+0x2fe/0x370
 [<000003ffe016d28e>] kvm_s390_vm_mem_op_cmpxchg+0x17e/0x350
 [<000003ffe0173284>] kvm_arch_vm_ioctl+0x354/0x6f0
 [<000003ffe015fedc>] kvm_vm_ioctl+0x2cc/0x6e0
 [<000003ffe05348ae>] vfs_ioctl+0x2e/0x70
 [<000003ffe0535e70>] __s390x_sys_ioctl+0xe0/0x100
 [<000003ffe0f40f06>] __do_syscall+0x136/0x340
 [<000003ffe0f4cb2e>] system_call+0x6e/0x90
Last Breaking-Event-Address:
 [<000003ffe0f4d896>] __cmpxchg_user_key1+0x36/0x190

Fix this by defining all code ranges within cmpxchg_user_key() functions
which may be executed with a non-default key and explicitly initialize
storage keys by calling skey_regions_initialize().

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>

1 files changed, 21 insertions, 5 deletions

diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index f6112c7dfcc7..02359c03b809 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -13,6 +13,7 @@
 #include <linux/mm.h>
 #include <asm/asm-extable.h>
 #include <asm/ctlreg.h>
+#include <asm/skey.h>
 
 #ifdef CONFIG_DEBUG_ENTRY
 void debug_user_asce(int exit)
@@ -156,6 +157,7 @@ int __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
 	bool sacf_flag;
 	int rc = 0;
 
+	skey_regions_initialize();
 	shift = (3 ^ (address & 3)) << 3;
 	address ^= address & 3;
 	_old = (unsigned int)old << shift;
@@ -163,7 +165,7 @@ int __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
 	mask = ~(0xff << shift);
 	sacf_flag = enable_sacf_uaccess();
 	asm_inline volatile(
-		"	spka	0(%[key])\n"
+		"20:	spka	0(%[key])\n"
 		"	sacf	256\n"
 		"	llill	%[count],%[max_loops]\n"
 		"0:	l	%[prev],%[address]\n"
@@ -181,10 +183,12 @@ int __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
 		"	brct	%[count],2b\n"
 		"5:	sacf	768\n"
 		"	spka	%[default_key]\n"
+		"21:\n"
 		EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
+		SKEY_REGION(20b, 21b)
 		: [rc] "+&d" (rc),
 		[prev] "=&d" (prev),
 		[address] "+Q" (*(int *)address),
@@ -212,6 +216,7 @@ int __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
 	bool sacf_flag;
 	int rc = 0;
 
+	skey_regions_initialize();
 	shift = (2 ^ (address & 2)) << 3;
 	address ^= address & 2;
 	_old = (unsigned int)old << shift;
@@ -219,7 +224,7 @@ int __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
 	mask = ~(0xffff << shift);
 	sacf_flag = enable_sacf_uaccess();
 	asm_inline volatile(
-		"	spka	0(%[key])\n"
+		"20:	spka	0(%[key])\n"
 		"	sacf	256\n"
 		"	llill	%[count],%[max_loops]\n"
 		"0:	l	%[prev],%[address]\n"
@@ -237,10 +242,12 @@ int __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
 		"	brct	%[count],2b\n"
 		"5:	sacf	768\n"
 		"	spka	%[default_key]\n"
+		"21:\n"
 		EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
+		SKEY_REGION(20b, 21b)
 		: [rc] "+&d" (rc),
 		[prev] "=&d" (prev),
 		[address] "+Q" (*(int *)address),
@@ -267,15 +274,18 @@ int __cmpxchg_user_key4(unsigned long address, unsigned int *uval,
 	bool sacf_flag;
 	int rc = 0;
 
+	skey_regions_initialize();
 	sacf_flag = enable_sacf_uaccess();
 	asm_inline volatile(
-		"	spka	0(%[key])\n"
+		"20:	spka	0(%[key])\n"
 		"	sacf	256\n"
 		"0:	cs	%[prev],%[new],%[address]\n"
 		"1:	sacf	768\n"
 		"	spka	%[default_key]\n"
+		"21:\n"
 		EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
+		SKEY_REGION(20b, 21b)
 		: [rc] "+&d" (rc),
 		[prev] "+&d" (prev),
 		[address] "+Q" (*(int *)address)
@@ -296,15 +306,18 @@ int __cmpxchg_user_key8(unsigned long address, unsigned long *uval,
 	bool sacf_flag;
 	int rc = 0;
 
+	skey_regions_initialize();
 	sacf_flag = enable_sacf_uaccess();
 	asm_inline volatile(
-		"	spka	0(%[key])\n"
+		"20:	spka	0(%[key])\n"
 		"	sacf	256\n"
 		"0:	csg	%[prev],%[new],%[address]\n"
 		"1:	sacf	768\n"
 		"	spka	%[default_key]\n"
+		"21:\n"
 		EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
+		SKEY_REGION(20b, 21b)
 		: [rc] "+&d" (rc),
 		[prev] "+&d" (prev),
 		[address] "+QS" (*(long *)address)
@@ -325,15 +338,18 @@ int __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
 	bool sacf_flag;
 	int rc = 0;
 
+	skey_regions_initialize();
 	sacf_flag = enable_sacf_uaccess();
 	asm_inline volatile(
-		"	spka	0(%[key])\n"
+		"20:	spka	0(%[key])\n"
 		"	sacf	256\n"
 		"0:	cdsg	%[prev],%[new],%[address]\n"
 		"1:	sacf	768\n"
 		"	spka	%[default_key]\n"
+		"21:\n"
 		EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
 		EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
+		SKEY_REGION(20b, 21b)
 		: [rc] "+&d" (rc),
 		[prev] "+&d" (prev),
 		[address] "+QS" (*(__int128_t *)address)