diff options
Diffstat (limited to 'arch/s390')
115 files changed, 3742 insertions, 2638 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2508a6f31588..ed5cb5af5281 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -2,7 +2,7 @@ config MMU def_bool y config ZONE_DMA - def_bool y if 64BIT + def_bool y config LOCKDEP_SUPPORT def_bool y @@ -81,6 +81,7 @@ config S390 select INIT_ALL_POSSIBLE select HAVE_IRQ_WORK select HAVE_PERF_EVENTS + select ARCH_HAVE_NMI_SAFE_CMPXCHG select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA @@ -88,6 +89,8 @@ config S390 select HAVE_KERNEL_XZ select HAVE_GET_USER_PAGES_FAST select HAVE_ARCH_MUTEX_CPU_RELAX + select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 + select HAVE_RCU_TABLE_FREE if SMP select ARCH_INLINE_SPIN_TRYLOCK select ARCH_INLINE_SPIN_TRYLOCK_BH select ARCH_INLINE_SPIN_LOCK @@ -229,17 +232,6 @@ config SYSVIPC_COMPAT config AUDIT_ARCH def_bool y -config S390_EXEC_PROTECT - def_bool y - prompt "Data execute protection" - help - This option allows to enable a buffer overflow protection for user - space programs and it also selects the addressing mode option above. - The kernel parameter noexec=on will enable this feature and also - switch the addressing modes, default is disabled. Enabling this (via - kernel parameter) on machines earlier than IBM System z9 this will - reduce system performance. - comment "Code generation options" choice @@ -282,11 +274,11 @@ config MARCH_Z10 on older machines. config MARCH_Z196 - bool "IBM zEnterprise 196" + bool "IBM zEnterprise 114 and 196" help - Select this to enable optimizations for IBM zEnterprise 196 - (2817 series). The kernel will be slightly faster but will not work - on older machines. + Select this to enable optimizations for IBM zEnterprise 114 and 196 + (2818 and 2817 series). The kernel will be slightly faster but will + not work on older machines. endchoice @@ -588,6 +580,7 @@ config S390_GUEST def_bool y prompt "s390 guest support for KVM (EXPERIMENTAL)" depends on 64BIT && EXPERIMENTAL + select VIRTUALIZATION select VIRTIO select VIRTIO_RING select VIRTIO_CONSOLE diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index 5c91995b74e4..24bff4f1cc52 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -130,9 +130,7 @@ static void appldata_work_fn(struct work_struct *work) { struct list_head *lh; struct appldata_ops *ops; - int i; - i = 0; get_online_cpus(); mutex_lock(&appldata_ops_mutex); list_for_each(lh, &appldata_ops_list) { diff --git a/arch/s390/appldata/appldata_mem.c b/arch/s390/appldata/appldata_mem.c index e43fe7537031..f7d3dc555bdb 100644 --- a/arch/s390/appldata/appldata_mem.c +++ b/arch/s390/appldata/appldata_mem.c @@ -92,9 +92,7 @@ static void appldata_get_mem_data(void *data) mem_data->pswpin = ev[PSWPIN]; mem_data->pswpout = ev[PSWPOUT]; mem_data->pgalloc = ev[PGALLOC_NORMAL]; -#ifdef CONFIG_ZONE_DMA mem_data->pgalloc += ev[PGALLOC_DMA]; -#endif mem_data->pgfault = ev[PGFAULT]; mem_data->pgmajfault = ev[PGMAJFAULT]; diff --git a/arch/s390/boot/compressed/head31.S b/arch/s390/boot/compressed/head31.S index 2a5523a32bcc..e8c9e18b8039 100644 --- a/arch/s390/boot/compressed/head31.S +++ b/arch/s390/boot/compressed/head31.S @@ -7,14 +7,14 @@ */ #include <linux/init.h> +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/page.h> #include "sizes.h" __HEAD - .globl startup_continue -startup_continue: +ENTRY(startup_continue) basr %r13,0 # get base .LPG1: # setup stack diff --git a/arch/s390/boot/compressed/head64.S b/arch/s390/boot/compressed/head64.S index 2982cb140550..f86a4eef28a9 100644 --- a/arch/s390/boot/compressed/head64.S +++ b/arch/s390/boot/compressed/head64.S @@ -7,14 +7,14 @@ */ #include <linux/init.h> +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/page.h> #include "sizes.h" __HEAD - .globl startup_continue -startup_continue: +ENTRY(startup_continue) basr %r13,0 # get base .LPG1: # setup stack diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 1cf81d77c5a5..7f0b7cda6259 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o +obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 58f46734465f..a9ce135893f8 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -31,7 +31,8 @@ #define AES_KEYLEN_192 2 #define AES_KEYLEN_256 4 -static char keylen_flag = 0; +static u8 *ctrblk; +static char keylen_flag; struct s390_aes_ctx { u8 iv[AES_BLOCK_SIZE]; @@ -45,6 +46,24 @@ struct s390_aes_ctx { } fallback; }; +struct pcc_param { + u8 key[32]; + u8 tweak[16]; + u8 block[16]; + u8 bit[16]; + u8 xts[16]; +}; + +struct s390_xts_ctx { + u8 key[32]; + u8 xts_param[16]; + struct pcc_param pcc; + long enc; + long dec; + int key_len; + struct crypto_blkcipher *fallback; +}; + /* * Check if the key_len is supported by the HW. * Returns 0 if it is, a positive number if it is not and software fallback is @@ -504,15 +523,337 @@ static struct crypto_alg cbc_aes_alg = { } }; +static int xts_fallback_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int len) +{ + struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); + unsigned int ret; + + xts_ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + xts_ctx->fallback->base.crt_flags |= (tfm->crt_flags & + CRYPTO_TFM_REQ_MASK); + + ret = crypto_blkcipher_setkey(xts_ctx->fallback, key, len); + if (ret) { + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= (xts_ctx->fallback->base.crt_flags & + CRYPTO_TFM_RES_MASK); + } + return ret; +} + +static int xts_fallback_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); + struct crypto_blkcipher *tfm; + unsigned int ret; + + tfm = desc->tfm; + desc->tfm = xts_ctx->fallback; + + ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); + + desc->tfm = tfm; + return ret; +} + +static int xts_fallback_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); + struct crypto_blkcipher *tfm; + unsigned int ret; + + tfm = desc->tfm; + desc->tfm = xts_ctx->fallback; + + ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); + + desc->tfm = tfm; + return ret; +} + +static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); + u32 *flags = &tfm->crt_flags; + + switch (key_len) { + case 32: + xts_ctx->enc = KM_XTS_128_ENCRYPT; + xts_ctx->dec = KM_XTS_128_DECRYPT; + memcpy(xts_ctx->key + 16, in_key, 16); + memcpy(xts_ctx->pcc.key + 16, in_key + 16, 16); + break; + case 48: + xts_ctx->enc = 0; + xts_ctx->dec = 0; + xts_fallback_setkey(tfm, in_key, key_len); + break; + case 64: + xts_ctx->enc = KM_XTS_256_ENCRYPT; + xts_ctx->dec = KM_XTS_256_DECRYPT; + memcpy(xts_ctx->key, in_key, 32); + memcpy(xts_ctx->pcc.key, in_key + 32, 32); + break; + default: + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + xts_ctx->key_len = key_len; + return 0; +} + +static int xts_aes_crypt(struct blkcipher_desc *desc, long func, + struct s390_xts_ctx *xts_ctx, + struct blkcipher_walk *walk) +{ + unsigned int offset = (xts_ctx->key_len >> 1) & 0x10; + int ret = blkcipher_walk_virt(desc, walk); + unsigned int nbytes = walk->nbytes; + unsigned int n; + u8 *in, *out; + void *param; + + if (!nbytes) + goto out; + + memset(xts_ctx->pcc.block, 0, sizeof(xts_ctx->pcc.block)); + memset(xts_ctx->pcc.bit, 0, sizeof(xts_ctx->pcc.bit)); + memset(xts_ctx->pcc.xts, 0, sizeof(xts_ctx->pcc.xts)); + memcpy(xts_ctx->pcc.tweak, walk->iv, sizeof(xts_ctx->pcc.tweak)); + param = xts_ctx->pcc.key + offset; + ret = crypt_s390_pcc(func, param); + BUG_ON(ret < 0); + + memcpy(xts_ctx->xts_param, xts_ctx->pcc.xts, 16); + param = xts_ctx->key + offset; + do { + /* only use complete blocks */ + n = nbytes & ~(AES_BLOCK_SIZE - 1); + out = walk->dst.virt.addr; + in = walk->src.virt.addr; + + ret = crypt_s390_km(func, param, out, in, n); + BUG_ON(ret < 0 || ret != n); + + nbytes &= AES_BLOCK_SIZE - 1; + ret = blkcipher_walk_done(desc, walk, nbytes); + } while ((nbytes = walk->nbytes)); +out: + return ret; +} + +static int xts_aes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + if (unlikely(xts_ctx->key_len == 48)) + return xts_fallback_encrypt(desc, dst, src, nbytes); + + blkcipher_walk_init(&walk, dst, src, nbytes); + return xts_aes_crypt(desc, xts_ctx->enc, xts_ctx, &walk); +} + +static int xts_aes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + if (unlikely(xts_ctx->key_len == 48)) + return xts_fallback_decrypt(desc, dst, src, nbytes); + + blkcipher_walk_init(&walk, dst, src, nbytes); + return xts_aes_crypt(desc, xts_ctx->dec, xts_ctx, &walk); +} + +static int xts_fallback_init(struct crypto_tfm *tfm) +{ + const char *name = tfm->__crt_alg->cra_name; + struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); + + xts_ctx->fallback = crypto_alloc_blkcipher(name, 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(xts_ctx->fallback)) { + pr_err("Allocating XTS fallback algorithm %s failed\n", + name); + return PTR_ERR(xts_ctx->fallback); + } + return 0; +} + +static void xts_fallback_exit(struct crypto_tfm *tfm) +{ + struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); + + crypto_free_blkcipher(xts_ctx->fallback); + xts_ctx->fallback = NULL; +} + +static struct crypto_alg xts_aes_alg = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-s390", + .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s390_xts_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(xts_aes_alg.cra_list), + .cra_init = xts_fallback_init, + .cra_exit = xts_fallback_exit, + .cra_u = { + .blkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = xts_aes_set_key, + .encrypt = xts_aes_encrypt, + .decrypt = xts_aes_decrypt, + } + } +}; + +static int ctr_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); + + switch (key_len) { + case 16: + sctx->enc = KMCTR_AES_128_ENCRYPT; + sctx->dec = KMCTR_AES_128_DECRYPT; + break; + case 24: + sctx->enc = KMCTR_AES_192_ENCRYPT; + sctx->dec = KMCTR_AES_192_DECRYPT; + break; + case 32: + sctx->enc = KMCTR_AES_256_ENCRYPT; + sctx->dec = KMCTR_AES_256_DECRYPT; + break; + } + + return aes_set_key(tfm, in_key, key_len); +} + +static int ctr_aes_crypt(struct blkcipher_desc *desc, long func, + struct s390_aes_ctx *sctx, struct blkcipher_walk *walk) +{ + int ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); + unsigned int i, n, nbytes; + u8 buf[AES_BLOCK_SIZE]; + u8 *out, *in; + + if (!walk->nbytes) + return ret; + + memcpy(ctrblk, walk->iv, AES_BLOCK_SIZE); + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + out = walk->dst.virt.addr; + in = walk->src.virt.addr; + while (nbytes >= AES_BLOCK_SIZE) { + /* only use complete blocks, max. PAGE_SIZE */ + n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : + nbytes & ~(AES_BLOCK_SIZE - 1); + for (i = AES_BLOCK_SIZE; i < n; i += AES_BLOCK_SIZE) { + memcpy(ctrblk + i, ctrblk + i - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(ctrblk + i, AES_BLOCK_SIZE); + } + ret = crypt_s390_kmctr(func, sctx->key, out, in, n, ctrblk); + BUG_ON(ret < 0 || ret != n); + if (n > AES_BLOCK_SIZE) + memcpy(ctrblk, ctrblk + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(ctrblk, AES_BLOCK_SIZE); + out += n; + in += n; + nbytes -= n; + } + ret = blkcipher_walk_done(desc, walk, nbytes); + } + /* + * final block may be < AES_BLOCK_SIZE, copy only nbytes + */ + if (nbytes) { + out = walk->dst.virt.addr; + in = walk->src.virt.addr; + ret = crypt_s390_kmctr(func, sctx->key, buf, in, + AES_BLOCK_SIZE, ctrblk); + BUG_ON(ret < 0 || ret != AES_BLOCK_SIZE); + memcpy(out, buf, nbytes); + crypto_inc(ctrblk, AES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, 0); + } + memcpy(walk->iv, ctrblk, AES_BLOCK_SIZE); + return ret; +} + +static int ctr_aes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_aes_crypt(desc, sctx->enc, sctx, &walk); +} + +static int ctr_aes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_aes_crypt(desc, sctx->dec, sctx, &walk); +} + +static struct crypto_alg ctr_aes_alg = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-s390", + .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct s390_aes_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ctr_aes_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ctr_aes_set_key, + .encrypt = ctr_aes_encrypt, + .decrypt = ctr_aes_decrypt, + } + } +}; + static int __init aes_s390_init(void) { int ret; - if (crypt_s390_func_available(KM_AES_128_ENCRYPT)) + if (crypt_s390_func_available(KM_AES_128_ENCRYPT, CRYPT_S390_MSA)) keylen_flag |= AES_KEYLEN_128; - if (crypt_s390_func_available(KM_AES_192_ENCRYPT)) + if (crypt_s390_func_available(KM_AES_192_ENCRYPT, CRYPT_S390_MSA)) keylen_flag |= AES_KEYLEN_192; - if (crypt_s390_func_available(KM_AES_256_ENCRYPT)) + if (crypt_s390_func_available(KM_AES_256_ENCRYPT, CRYPT_S390_MSA)) keylen_flag |= AES_KEYLEN_256; if (!keylen_flag) @@ -535,9 +876,40 @@ static int __init aes_s390_init(void) if (ret) goto cbc_aes_err; + if (crypt_s390_func_available(KM_XTS_128_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4) && + crypt_s390_func_available(KM_XTS_256_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + ret = crypto_register_alg(&xts_aes_alg); + if (ret) + goto xts_aes_err; + } + + if (crypt_s390_func_available(KMCTR_AES_128_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4) && + crypt_s390_func_available(KMCTR_AES_192_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4) && + crypt_s390_func_available(KMCTR_AES_256_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + ctrblk = (u8 *) __get_free_page(GFP_KERNEL); + if (!ctrblk) { + ret = -ENOMEM; + goto ctr_aes_err; + } + ret = crypto_register_alg(&ctr_aes_alg); + if (ret) { + free_page((unsigned long) ctrblk); + goto ctr_aes_err; + } + } + out: return ret; +ctr_aes_err: + crypto_unregister_alg(&xts_aes_alg); +xts_aes_err: + crypto_unregister_alg(&cbc_aes_alg); cbc_aes_err: crypto_unregister_alg(&ecb_aes_alg); ecb_aes_err: @@ -548,6 +920,9 @@ aes_err: static void __exit aes_s390_fini(void) { + crypto_unregister_alg(&ctr_aes_alg); + free_page((unsigned long) ctrblk); + crypto_unregister_alg(&xts_aes_alg); crypto_unregister_alg(&cbc_aes_alg); crypto_unregister_alg(&ecb_aes_alg); crypto_unregister_alg(&aes_alg); diff --git a/arch/s390/crypto/crypt_s390.h b/arch/s390/crypto/crypt_s390.h index 7ee9a1b4ad9f..49676771bd66 100644 --- a/arch/s390/crypto/crypt_s390.h +++ b/arch/s390/crypto/crypt_s390.h @@ -24,13 +24,18 @@ #define CRYPT_S390_PRIORITY 300 #define CRYPT_S390_COMPOSITE_PRIORITY 400 +#define CRYPT_S390_MSA 0x1 +#define CRYPT_S390_MSA3 0x2 +#define CRYPT_S390_MSA4 0x4 + /* s390 cryptographic operations */ enum crypt_s390_operations { CRYPT_S390_KM = 0x0100, CRYPT_S390_KMC = 0x0200, CRYPT_S390_KIMD = 0x0300, CRYPT_S390_KLMD = 0x0400, - CRYPT_S390_KMAC = 0x0500 + CRYPT_S390_KMAC = 0x0500, + CRYPT_S390_KMCTR = 0x0600 }; /* @@ -51,6 +56,10 @@ enum crypt_s390_km_func { KM_AES_192_DECRYPT = CRYPT_S390_KM | 0x13 | 0x80, KM_AES_256_ENCRYPT = CRYPT_S390_KM | 0x14, KM_AES_256_DECRYPT = CRYPT_S390_KM | 0x14 | 0x80, + KM_XTS_128_ENCRYPT = CRYPT_S390_KM | 0x32, + KM_XTS_128_DECRYPT = CRYPT_S390_KM | 0x32 | 0x80, + KM_XTS_256_ENCRYPT = CRYPT_S390_KM | 0x34, + KM_XTS_256_DECRYPT = CRYPT_S390_KM | 0x34 | 0x80, }; /* @@ -75,6 +84,26 @@ enum crypt_s390_kmc_func { }; /* + * function codes for KMCTR (CIPHER MESSAGE WITH COUNTER) + * instruction + */ +enum crypt_s390_kmctr_func { + KMCTR_QUERY = CRYPT_S390_KMCTR | 0x0, + KMCTR_DEA_ENCRYPT = CRYPT_S390_KMCTR | 0x1, + KMCTR_DEA_DECRYPT = CRYPT_S390_KMCTR | 0x1 | 0x80, + KMCTR_TDEA_128_ENCRYPT = CRYPT_S390_KMCTR | 0x2, + KMCTR_TDEA_128_DECRYPT = CRYPT_S390_KMCTR | 0x2 | 0x80, + KMCTR_TDEA_192_ENCRYPT = CRYPT_S390_KMCTR | 0x3, + KMCTR_TDEA_192_DECRYPT = CRYPT_S390_KMCTR | 0x3 | 0x80, + KMCTR_AES_128_ENCRYPT = CRYPT_S390_KMCTR | 0x12, + KMCTR_AES_128_DECRYPT = CRYPT_S390_KMCTR | 0x12 | 0x80, + KMCTR_AES_192_ENCRYPT = CRYPT_S390_KMCTR | 0x13, + KMCTR_AES_192_DECRYPT = CRYPT_S390_KMCTR | 0x13 | 0x80, + KMCTR_AES_256_ENCRYPT = CRYPT_S390_KMCTR | 0x14, + KMCTR_AES_256_DECRYPT = CRYPT_S390_KMCTR | 0x14 | 0x80, +}; + +/* * function codes for KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) * instruction */ @@ -83,6 +112,7 @@ enum crypt_s390_kimd_func { KIMD_SHA_1 = CRYPT_S390_KIMD | 1, KIMD_SHA_256 = CRYPT_S390_KIMD | 2, KIMD_SHA_512 = CRYPT_S390_KIMD | 3, + KIMD_GHASH = CRYPT_S390_KIMD | 65, }; /* @@ -284,6 +314,45 @@ static inline int crypt_s390_kmac(long func, void *param, } /** + * crypt_s390_kmctr: + * @func: the function code passed to KMCTR; see crypt_s390_kmctr_func + * @param: address of parameter block; see POP for details on each func + * @dest: address of destination memory area + * @src: address of source memory area + * @src_len: length of src operand in bytes + * @counter: address of counter value + * + * Executes the KMCTR (CIPHER MESSAGE WITH COUNTER) operation of the CPU. + * + * Returns -1 for failure, 0 for the query func, number of processed + * bytes for encryption/decryption funcs + */ +static inline int crypt_s390_kmctr(long func, void *param, u8 *dest, + const u8 *src, long src_len, u8 *counter) +{ + register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; + register void *__param asm("1") = param; + register const u8 *__src asm("2") = src; + register long __src_len asm("3") = src_len; + register u8 *__dest asm("4") = dest; + register u8 *__ctr asm("6") = counter; + int ret = -1; + + asm volatile( + "0: .insn rrf,0xb92d0000,%3,%1,%4,0 \n" /* KMCTR opcode */ + "1: brc 1,0b \n" /* handle partial completion */ + " la %0,0\n" + "2:\n" + EX_TABLE(0b,2b) EX_TABLE(1b,2b) + : "+d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest), + "+a" (__ctr) + : "d" (__func), "a" (__param) : "cc", "memory"); + if (ret < 0) + return ret; + return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; +} + +/** * crypt_s390_func_available: * @func: the function code of the specific function; 0 if op in general * @@ -291,13 +360,17 @@ static inline int crypt_s390_kmac(long func, void *param, * * Returns 1 if func available; 0 if func or op in general not available */ -static inline int crypt_s390_func_available(int func) +static inline int crypt_s390_func_available(int func, + unsigned int facility_mask) { unsigned char status[16]; int ret; - /* check if CPACF facility (bit 17) is available */ - if (!test_facility(17)) + if (facility_mask & CRYPT_S390_MSA && !test_facility(17)) + return 0; + if (facility_mask & CRYPT_S390_MSA3 && !test_facility(76)) + return 0; + if (facility_mask & CRYPT_S390_MSA4 && !test_facility(77)) return 0; switch (func & CRYPT_S390_OP_MASK) { @@ -316,6 +389,10 @@ static inline int crypt_s390_func_available(int func) case CRYPT_S390_KMAC: ret = crypt_s390_kmac(KMAC_QUERY, &status, NULL, 0); break; + case CRYPT_S390_KMCTR: + ret = crypt_s390_kmctr(KMCTR_QUERY, &status, NULL, NULL, 0, + NULL); + break; default: return 0; } @@ -326,4 +403,31 @@ static inline int crypt_s390_func_available(int func) return (status[func >> 3] & (0x80 >> (func & 7))) != 0; } +/** + * crypt_s390_pcc: + * @func: the function code passed to KM; see crypt_s390_km_func + * @param: address of parameter block; see POP for details on each func + * + * Executes the PCC (PERFORM CRYPTOGRAPHIC COMPUTATION) operation of the CPU. + * + * Returns -1 for failure, 0 for success. + */ +static inline int crypt_s390_pcc(long func, void *param) +{ + register long __func asm("0") = func & 0x7f; /* encrypt or decrypt */ + register void *__param asm("1") = param; + int ret = -1; + + asm volatile( + "0: .insn rre,0xb92c0000,0,0 \n" /* PCC opcode */ + "1: brc 1,0b \n" /* handle partial completion */ + " la %0,0\n" + "2:\n" + EX_TABLE(0b,2b) EX_TABLE(1b,2b) + : "+d" (ret) + : "d" (__func), "a" (__param) : "cc", "memory"); + return ret; +} + + #endif /* _CRYPTO_ARCH_S390_CRYPT_S390_H */ diff --git a/arch/s390/crypto/des_check_key.c b/arch/s390/crypto/des_check_key.c deleted file mode 100644 index 5706af266442..000000000000 --- a/arch/s390/crypto/des_check_key.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Cryptographic API. - * - * Function for checking keys for the DES and Tripple DES Encryption - * algorithms. - * - * Originally released as descore by Dana L. How <how@isl.stanford.edu>. - * Modified by Raimar Falke <rf13@inf.tu-dresden.de> for the Linux-Kernel. - * Derived from Cryptoapi and Nettle implementations, adapted for in-place - * scatterlist interface. Changed LGPL to GPL per section 3 of the LGPL. - * - * s390 Version: - * Copyright IBM Corp. 2003 - * Author(s): Thomas Spatzier - * Jan Glauber (jan.glauber@de.ibm.com) - * - * Derived from "crypto/des.c" - * Copyright (c) 1992 Dana L. How. - * Copyright (c) Raimar Falke <rf13@inf.tu-dresden.de> - * Copyright (c) Gisle Sflensminde <gisle@ii.uib.no> - * Copyright (C) 2001 Niels Mvller. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - */ -#include <linux/init.h> -#include <linux/module.h> -#include <linux/errno.h> -#include <linux/crypto.h> -#include "crypto_des.h" - -#define ROR(d,c,o) ((d) = (d) >> (c) | (d) << (o)) - -static const u8 parity[] = { - 8,1,0,8,0,8,8,0,0,8,8,0,8,0,2,8,0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,3, - 0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0,8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8, - 0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0,8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8, - 8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8,0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0, - 0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0,8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8, - 8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8,0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0, - 8,0,0,8,0,8,8,0,0,8,8,0,8,0,0,8,0,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0, - 4,8,8,0,8,0,0,8,8,0,0,8,0,8,8,0,8,5,0,8,0,8,8,0,0,8,8,0,8,0,6,8, -}; - -/* - * RFC2451: Weak key checks SHOULD be performed. - */ -int -crypto_des_check_key(const u8 *key, unsigned int keylen, u32 *flags) -{ - u32 n, w; - - n = parity[key[0]]; n <<= 4; - n |= parity[key[1]]; n <<= 4; - n |= parity[key[2]]; n <<= 4; - n |= parity[key[3]]; n <<= 4; - n |= parity[key[4]]; n <<= 4; - n |= parity[key[5]]; n <<= 4; - n |= parity[key[6]]; n <<= 4; - n |= parity[key[7]]; - w = 0x88888888L; - - if ((*flags & CRYPTO_TFM_REQ_WEAK_KEY) - && !((n - (w >> 3)) & w)) { /* 1 in 10^10 keys passes this test */ - if (n < 0x41415151) { - if (n < 0x31312121) { - if (n < 0x14141515) { - /* 01 01 01 01 01 01 01 01 */ - if (n == 0x11111111) goto weak; - /* 01 1F 01 1F 01 0E 01 0E */ - if (n == 0x13131212) goto weak; - } else { - /* 01 E0 01 E0 01 F1 01 F1 */ - if (n == 0x14141515) goto weak; - /* 01 FE 01 FE 01 FE 01 FE */ - if (n == 0x16161616) goto weak; - } - } else { - if (n < 0x34342525) { - /* 1F 01 1F 01 0E 01 0E 01 */ - if (n == 0x31312121) goto weak; - /* 1F 1F 1F 1F 0E 0E 0E 0E (?) */ - if (n == 0x33332222) goto weak; - } else { - /* 1F E0 1F E0 0E F1 0E F1 */ - if (n == 0x34342525) goto weak; - /* 1F FE 1F FE 0E FE 0E FE */ - if (n == 0x36362626) goto weak; - } - } - } else { - if (n < 0x61616161) { - if (n < 0x44445555) { - /* E0 01 E0 01 F1 01 F1 01 */ - if (n == 0x41415151) goto weak; - /* E0 1F E0 1F F1 0E F1 0E */ - if (n == 0x43435252) goto weak; - } else { - /* E0 E0 E0 E0 F1 F1 F1 F1 (?) */ - if (n == 0x44445555) goto weak; - /* E0 FE E0 FE F1 FE F1 FE */ - if (n == 0x46465656) goto weak; - } - } else { - if (n < 0x64646565) { - /* FE 01 FE 01 FE 01 FE 01 */ - if (n == 0x61616161) goto weak; - /* FE 1F FE 1F FE 0E FE 0E */ - if (n == 0x63636262) goto weak; - } else { - /* FE E0 FE E0 FE F1 FE F1 */ - if (n == 0x64646565) goto weak; - /* FE FE FE FE FE FE FE FE */ - if (n == 0x66666666) goto weak; - } - } - } - } - return 0; -weak: - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; -} - -EXPORT_SYMBOL(crypto_des_check_key); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Key Check function for DES & DES3 Cipher Algorithms"); diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index cc5420118393..a52bfd124d86 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -3,7 +3,7 @@ * * s390 implementation of the DES Cipher Algorithm. * - * Copyright IBM Corp. 2003,2007 + * Copyright IBM Corp. 2003,2011 * Author(s): Thomas Spatzier * Jan Glauber (jan.glauber@de.ibm.com) * @@ -22,22 +22,19 @@ #include "crypt_s390.h" -#define DES3_192_KEY_SIZE (3 * DES_KEY_SIZE) +#define DES3_KEY_SIZE (3 * DES_KEY_SIZE) -struct crypt_s390_des_ctx { - u8 iv[DES_BLOCK_SIZE]; - u8 key[DES_KEY_SIZE]; -}; +static u8 *ctrblk; -struct crypt_s390_des3_192_ctx { +struct s390_des_ctx { u8 iv[DES_BLOCK_SIZE]; - u8 key[DES3_192_KEY_SIZE]; + u8 key[DES3_KEY_SIZE]; }; static int des_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) + unsigned int key_len) { - struct crypt_s390_des_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; u32 tmp[DES_EXPKEY_WORDS]; @@ -47,22 +44,22 @@ static int des_setkey(struct crypto_tfm *tfm, const u8 *key, return -EINVAL; } - memcpy(dctx->key, key, keylen); + memcpy(ctx->key, key, key_len); return 0; } static void des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { - struct crypt_s390_des_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_DEA_ENCRYPT, dctx->key, out, in, DES_BLOCK_SIZE); + crypt_s390_km(KM_DEA_ENCRYPT, ctx->key, out, in, DES_BLOCK_SIZE); } static void des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { - struct crypt_s390_des_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_DEA_DECRYPT, dctx->key, out, in, DES_BLOCK_SIZE); + crypt_s390_km(KM_DEA_DECRYPT, ctx->key, out, in, DES_BLOCK_SIZE); } static struct crypto_alg des_alg = { @@ -71,7 +68,7 @@ static struct crypto_alg des_alg = { .cra_priority = CRYPT_S390_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(des_alg.cra_list), .cra_u = { @@ -86,7 +83,7 @@ static struct crypto_alg des_alg = { }; static int ecb_desall_crypt(struct blkcipher_desc *desc, long func, - void *param, struct blkcipher_walk *walk) + u8 *key, struct blkcipher_walk *walk) { int ret = blkcipher_walk_virt(desc, walk); unsigned int nbytes; @@ -97,7 +94,7 @@ static int ecb_desall_crypt(struct blkcipher_desc *desc, long func, u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_km(func, param, out, in, n); + ret = crypt_s390_km(func, key, out, in, n); BUG_ON((ret < 0) || (ret != n)); nbytes &= DES_BLOCK_SIZE - 1; @@ -108,7 +105,7 @@ static int ecb_desall_crypt(struct blkcipher_desc *desc, long func, } static int cbc_desall_crypt(struct blkcipher_desc *desc, long func, - void *param, struct blkcipher_walk *walk) + u8 *iv, struct blkcipher_walk *walk) { int ret = blkcipher_walk_virt(desc, walk); unsigned int nbytes = walk->nbytes; @@ -116,20 +113,20 @@ static int cbc_desall_crypt(struct blkcipher_desc *desc, long func, if (!nbytes) goto out; - memcpy(param, walk->iv, DES_BLOCK_SIZE); + memcpy(iv, walk->iv, DES_BLOCK_SIZE); do { /* only use complete blocks */ unsigned int n = nbytes & ~(DES_BLOCK_SIZE - 1); u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_kmc(func, param, out, in, n); + ret = crypt_s390_kmc(func, iv, out, in, n); BUG_ON((ret < 0) || (ret != n)); nbytes &= DES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, walk, nbytes); } while ((nbytes = walk->nbytes)); - memcpy(walk->iv, param, DES_BLOCK_SIZE); + memcpy(walk->iv, iv, DES_BLOCK_SIZE); out: return ret; @@ -139,22 +136,22 @@ static int ecb_des_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct crypt_s390_des_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_DEA_ENCRYPT, sctx->key, &walk); + return ecb_desall_crypt(desc, KM_DEA_ENCRYPT, ctx->key, &walk); } static int ecb_des_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct crypt_s390_des_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_DEA_DECRYPT, sctx->key, &walk); + return ecb_desall_crypt(desc, KM_DEA_DECRYPT, ctx->key, &walk); } static struct crypto_alg ecb_des_alg = { @@ -163,7 +160,7 @@ static struct crypto_alg ecb_des_alg = { .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(ecb_des_alg.cra_list), @@ -182,22 +179,22 @@ static int cbc_des_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct crypt_s390_des_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_DEA_ENCRYPT, sctx->iv, &walk); + return cbc_desall_crypt(desc, KMC_DEA_ENCRYPT, ctx->iv, &walk); } static int cbc_des_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct crypt_s390_des_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_DEA_DECRYPT, sctx->iv, &walk); + return cbc_desall_crypt(desc, KMC_DEA_DECRYPT, ctx->iv, &walk); } static struct crypto_alg cbc_des_alg = { @@ -206,7 +203,7 @@ static struct crypto_alg cbc_des_alg = { .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(cbc_des_alg.cra_list), @@ -235,10 +232,10 @@ static struct crypto_alg cbc_des_alg = { * property. * */ -static int des3_192_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) +static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int key_len) { - struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && @@ -248,141 +245,276 @@ static int des3_192_setkey(struct crypto_tfm *tfm, const u8 *key, *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } - memcpy(dctx->key, key, keylen); + memcpy(ctx->key, key, key_len); return 0; } -static void des3_192_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void des3_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { - struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_TDEA_192_ENCRYPT, dctx->key, dst, (void*)src, - DES_BLOCK_SIZE); + crypt_s390_km(KM_TDEA_192_ENCRYPT, ctx->key, dst, src, DES_BLOCK_SIZE); } -static void des3_192_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void des3_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { - struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); + struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_TDEA_192_DECRYPT, dctx->key, dst, (void*)src, - DES_BLOCK_SIZE); + crypt_s390_km(KM_TDEA_192_DECRYPT, ctx->key, dst, src, DES_BLOCK_SIZE); } -static struct crypto_alg des3_192_alg = { +static struct crypto_alg des3_alg = { .cra_name = "des3_ede", .cra_driver_name = "des3_ede-s390", .cra_priority = CRYPT_S390_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(des3_192_alg.cra_list), + .cra_list = LIST_HEAD_INIT(des3_alg.cra_list), .cra_u = { .cipher = { - .cia_min_keysize = DES3_192_KEY_SIZE, - .cia_max_keysize = DES3_192_KEY_SIZE, - .cia_setkey = des3_192_setkey, - .cia_encrypt = des3_192_encrypt, - .cia_decrypt = des3_192_decrypt, + .cia_min_keysize = DES3_KEY_SIZE, + .cia_max_keysize = DES3_KEY_SIZE, + .cia_setkey = des3_setkey, + .cia_encrypt = des3_encrypt, + .cia_decrypt = des3_decrypt, } } }; -static int ecb_des3_192_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int ecb_des3_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) { - struct crypt_s390_des3_192_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_192_ENCRYPT, sctx->key, &walk); + return ecb_desall_crypt(desc, KM_TDEA_192_ENCRYPT, ctx->key, &walk); } -static int ecb_des3_192_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int ecb_des3_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) { - struct crypt_s390_des3_192_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_192_DECRYPT, sctx->key, &walk); + return ecb_desall_crypt(desc, KM_TDEA_192_DECRYPT, ctx->key, &walk); } -static struct crypto_alg ecb_des3_192_alg = { +static struct crypto_alg ecb_des3_alg = { .cra_name = "ecb(des3_ede)", .cra_driver_name = "ecb-des3_ede-s390", .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT( - ecb_des3_192_alg.cra_list), + ecb_des3_alg.cra_list), .cra_u = { .blkcipher = { - .min_keysize = DES3_192_KEY_SIZE, - .max_keysize = DES3_192_KEY_SIZE, - .setkey = des3_192_setkey, - .encrypt = ecb_des3_192_encrypt, - .decrypt = ecb_des3_192_decrypt, + .min_keysize = DES3_KEY_SIZE, + .max_keysize = DES3_KEY_SIZE, + .setkey = des3_setkey, + .encrypt = ecb_des3_encrypt, + .decrypt = ecb_des3_decrypt, } } }; -static int cbc_des3_192_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int cbc_des3_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) { - struct crypt_s390_des3_192_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_192_ENCRYPT, sctx->iv, &walk); + return cbc_desall_crypt(desc, KMC_TDEA_192_ENCRYPT, ctx->iv, &walk); } -static int cbc_des3_192_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int cbc_des3_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) { - struct crypt_s390_des3_192_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_192_DECRYPT, sctx->iv, &walk); + return cbc_desall_crypt(desc, KMC_TDEA_192_DECRYPT, ctx->iv, &walk); } -static struct crypto_alg cbc_des3_192_alg = { +static struct crypto_alg cbc_des3_alg = { .cra_name = "cbc(des3_ede)", .cra_driver_name = "cbc-des3_ede-s390", .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), + .cra_ctxsize = sizeof(struct s390_des_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT( - cbc_des3_192_alg.cra_list), + cbc_des3_alg.cra_list), .cra_u = { .blkcipher = { - .min_keysize = DES3_192_KEY_SIZE, - .max_keysize = DES3_192_KEY_SIZE, + .min_keysize = DES3_KEY_SIZE, + .max_keysize = DES3_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, - .setkey = des3_192_setkey, - .encrypt = cbc_des3_192_encrypt, - .decrypt = cbc_des3_192_decrypt, + .setkey = des3_setkey, + .encrypt = cbc_des3_encrypt, + .decrypt = cbc_des3_decrypt, } } }; -static int des_s390_init(void) +static int ctr_desall_crypt(struct blkcipher_desc *desc, long func, + struct s390_des_ctx *ctx, struct blkcipher_walk *walk) +{ + int ret = blkcipher_walk_virt_block(desc, walk, DES_BLOCK_SIZE); + unsigned int i, n, nbytes; + u8 buf[DES_BLOCK_SIZE]; + u8 *out, *in; + + memcpy(ctrblk, walk->iv, DES_BLOCK_SIZE); + while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) { + out = walk->dst.virt.addr; + in = walk->src.virt.addr; + while (nbytes >= DES_BLOCK_SIZE) { + /* align to block size, max. PAGE_SIZE */ + n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : + nbytes & ~(DES_BLOCK_SIZE - 1); + for (i = DES_BLOCK_SIZE; i < n; i += DES_BLOCK_SIZE) { + memcpy(ctrblk + i, ctrblk + i - DES_BLOCK_SIZE, + DES_BLOCK_SIZE); + crypto_inc(ctrblk + i, DES_BLOCK_SIZE); + } + ret = crypt_s390_kmctr(func, ctx->key, out, in, n, ctrblk); + BUG_ON((ret < 0) || (ret != n)); + if (n > DES_BLOCK_SIZE) + memcpy(ctrblk, ctrblk + n - DES_BLOCK_SIZE, + DES_BLOCK_SIZE); + crypto_inc(ctrblk, DES_BLOCK_SIZE); + out += n; + in += n; + nbytes -= n; + } + ret = blkcipher_walk_done(desc, walk, nbytes); + } + + /* final block may be < DES_BLOCK_SIZE, copy only nbytes */ + if (nbytes) { + out = walk->dst.virt.addr; + in = walk->src.virt.addr; + ret = crypt_s390_kmctr(func, ctx->key, buf, in, + DES_BLOCK_SIZE, ctrblk); + BUG_ON(ret < 0 || ret != DES_BLOCK_SIZE); + memcpy(out, buf, nbytes); + crypto_inc(ctrblk, DES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, 0); + } + memcpy(walk->iv, ctrblk, DES_BLOCK_SIZE); + return ret; +} + +static int ctr_des_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_desall_crypt(desc, KMCTR_DEA_ENCRYPT, ctx, &walk); +} + +static int ctr_des_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_desall_crypt(desc, KMCTR_DEA_DECRYPT, ctx, &walk); +} + +static struct crypto_alg ctr_des_alg = { + .cra_name = "ctr(des)", + .cra_driver_name = "ctr-des-s390", + .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct s390_des_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ctr_des_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = des_setkey, + .encrypt = ctr_des_encrypt, + .decrypt = ctr_des_decrypt, + } + } +}; + +static int ctr_des3_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_desall_crypt(desc, KMCTR_TDEA_192_ENCRYPT, ctx, &walk); +} + +static int ctr_des3_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_desall_crypt(desc, KMCTR_TDEA_192_DECRYPT, ctx, &walk); +} + +static struct crypto_alg ctr_des3_alg = { + .cra_name = "ctr(des3_ede)", + .cra_driver_name = "ctr-des3_ede-s390", + .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct s390_des_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ctr_des3_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = DES3_KEY_SIZE, + .max_keysize = DES3_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = des3_setkey, + .encrypt = ctr_des3_encrypt, + .decrypt = ctr_des3_decrypt, + } + } +}; + +static int __init des_s390_init(void) { int ret; - if (!crypt_s390_func_available(KM_DEA_ENCRYPT) || - !crypt_s390_func_available(KM_TDEA_192_ENCRYPT)) + if (!crypt_s390_func_available(KM_DEA_ENCRYPT, CRYPT_S390_MSA) || + !crypt_s390_func_available(KM_TDEA_192_ENCRYPT, CRYPT_S390_MSA)) return -EOPNOTSUPP; ret = crypto_register_alg(&des_alg); @@ -394,23 +526,46 @@ static int des_s390_init(void) ret = crypto_register_alg(&cbc_des_alg); if (ret) goto cbc_des_err; - ret = crypto_register_alg(&des3_192_alg); + ret = crypto_register_alg(&des3_alg); if (ret) - goto des3_192_err; - ret = crypto_register_alg(&ecb_des3_192_alg); + goto des3_err; + ret = crypto_register_alg(&ecb_des3_alg); if (ret) - goto ecb_des3_192_err; - ret = crypto_register_alg(&cbc_des3_192_alg); + goto ecb_des3_err; + ret = crypto_register_alg(&cbc_des3_alg); if (ret) - goto cbc_des3_192_err; + goto cbc_des3_err; + + if (crypt_s390_func_available(KMCTR_DEA_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4) && + crypt_s390_func_available(KMCTR_TDEA_192_ENCRYPT, + CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + ret = crypto_register_alg(&ctr_des_alg); + if (ret) + goto ctr_des_err; + ret = crypto_register_alg(&ctr_des3_alg); + if (ret) + goto ctr_des3_err; + ctrblk = (u8 *) __get_free_page(GFP_KERNEL); + if (!ctrblk) { + ret = -ENOMEM; + goto ctr_mem_err; + } + } out: return ret; -cbc_des3_192_err: - crypto_unregister_alg(&ecb_des3_192_alg); -ecb_des3_192_err: - crypto_unregister_alg(&des3_192_alg); -des3_192_err: +ctr_mem_err: + crypto_unregister_alg(&ctr_des3_alg); +ctr_des3_err: + crypto_unregister_alg(&ctr_des_alg); +ctr_des_err: + crypto_unregister_alg(&cbc_des3_alg); +cbc_des3_err: + crypto_unregister_alg(&ecb_des3_alg); +ecb_des3_err: + crypto_unregister_alg(&des3_alg); +des3_err: crypto_unregister_alg(&cbc_des_alg); cbc_des_err: crypto_unregister_alg(&ecb_des_alg); @@ -422,9 +577,14 @@ des_err: static void __exit des_s390_exit(void) { - crypto_unregister_alg(&cbc_des3_192_alg); - crypto_unregister_alg(&ecb_des3_192_alg); - crypto_unregister_alg(&des3_192_alg); + if (ctrblk) { + crypto_unregister_alg(&ctr_des_alg); + crypto_unregister_alg(&ctr_des3_alg); + free_page((unsigned long) ctrblk); + } + crypto_unregister_alg(&cbc_des3_alg); + crypto_unregister_alg(&ecb_des3_alg); + crypto_unregister_alg(&des3_alg); crypto_unregister_alg(&cbc_des_alg); crypto_unregister_alg(&ecb_des_alg); crypto_unregister_alg(&des_alg); diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c new file mode 100644 index 000000000000..b1bd170f24b1 --- /dev/null +++ b/arch/s390/crypto/ghash_s390.c @@ -0,0 +1,162 @@ +/* + * Cryptographic API. + * + * s390 implementation of the GHASH algorithm for GCM (Galois/Counter Mode). + * + * Copyright IBM Corp. 2011 + * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> + */ + +#include <crypto/internal/hash.h> +#include <linux/module.h> + +#include "crypt_s390.h" + +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 + +struct ghash_ctx { + u8 icv[16]; + u8 key[16]; +}; + +struct ghash_desc_ctx { + u8 buffer[GHASH_BLOCK_SIZE]; + u32 bytes; +}; + +static int ghash_init(struct shash_desc *desc) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int ghash_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + + if (keylen != GHASH_BLOCK_SIZE) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->key, key, GHASH_BLOCK_SIZE); + memset(ctx->icv, 0, GHASH_BLOCK_SIZE); + + return 0; +} + +static int ghash_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + unsigned int n; + u8 *buf = dctx->buffer; + int ret; + + if (dctx->bytes) { + u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); + + n = min(srclen, dctx->bytes); + dctx->bytes -= n; + srclen -= n; + + memcpy(pos, src, n); + src += n; + + if (!dctx->bytes) { + ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, + GHASH_BLOCK_SIZE); + BUG_ON(ret != GHASH_BLOCK_SIZE); + } + } + + n = srclen & ~(GHASH_BLOCK_SIZE - 1); + if (n) { + ret = crypt_s390_kimd(KIMD_GHASH, ctx, src, n); + BUG_ON(ret != n); + src += n; + srclen -= n; + } + + if (srclen) { + dctx->bytes = GHASH_BLOCK_SIZE - srclen; + memcpy(buf, src, srclen); + } + + return 0; +} + +static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +{ + u8 *buf = dctx->buffer; + int ret; + + if (dctx->bytes) { + u8 *pos = buf + (GHASH_BLOCK_SIZE - dctx->bytes); + + memset(pos, 0, dctx->bytes); + + ret = crypt_s390_kimd(KIMD_GHASH, ctx, buf, GHASH_BLOCK_SIZE); + BUG_ON(ret != GHASH_BLOCK_SIZE); + } + + dctx->bytes = 0; +} + +static int ghash_final(struct shash_desc *desc, u8 *dst) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + + ghash_flush(ctx, dctx); + memcpy(dst, ctx->icv, GHASH_BLOCK_SIZE); + + return 0; +} + +static struct shash_alg ghash_alg = { + .digestsize = GHASH_DIGEST_SIZE, + .init = ghash_init, + .update = ghash_update, + .final = ghash_final, + .setkey = ghash_setkey, + .descsize = sizeof(struct ghash_desc_ctx), + .base = { + .cra_name = "ghash", + .cra_driver_name = "ghash-s390", + .cra_priority = CRYPT_S390_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), + }, +}; + +static int __init ghash_mod_init(void) +{ + if (!crypt_s390_func_available(KIMD_GHASH, + CRYPT_S390_MSA | CRYPT_S390_MSA4)) + return -EOPNOTSUPP; + + return crypto_register_shash(&ghash_alg); +} + +static void __exit ghash_mod_exit(void) +{ + crypto_unregister_shash(&ghash_alg); +} + +module_init(ghash_mod_init); +module_exit(ghash_mod_exit); + +MODULE_ALIAS("ghash"); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("GHASH Message Digest Algorithm, s390 implementation"); diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 8b16c479585b..0808fbf0f7d3 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -166,7 +166,7 @@ static int __init prng_init(void) int ret; /* check if the CPU has a PRNG */ - if (!crypt_s390_func_available(KMC_PRNG)) + if (!crypt_s390_func_available(KMC_PRNG, CRYPT_S390_MSA)) return -EOPNOTSUPP; if (prng_chunk_size < 8) diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index f6de7826c979..e9868c6e0a08 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -90,7 +90,7 @@ static struct shash_alg alg = { static int __init sha1_s390_init(void) { - if (!crypt_s390_func_available(KIMD_SHA_1)) + if (!crypt_s390_func_available(KIMD_SHA_1, CRYPT_S390_MSA)) return -EOPNOTSUPP; return crypto_register_shash(&alg); } diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 61a7db372121..0317a3547cb9 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -1,15 +1,12 @@ /* * Cryptographic API. * - * s390 implementation of the SHA256 Secure Hash Algorithm. + * s390 implementation of the SHA256 and SHA224 Secure Hash Algorithm. * * s390 Version: - * Copyright IBM Corp. 2005,2007 + * Copyright IBM Corp. 2005,2011 * Author(s): Jan Glauber (jang@de.ibm.com) * - * Derived from "crypto/sha256_generic.c" - * and "arch/s390/crypto/sha1_s390.c" - * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) @@ -65,7 +62,7 @@ static int sha256_import(struct shash_desc *desc, const void *in) return 0; } -static struct shash_alg alg = { +static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, .update = s390_sha_update, @@ -84,22 +81,69 @@ static struct shash_alg alg = { } }; -static int sha256_s390_init(void) +static int sha224_init(struct shash_desc *desc) { - if (!crypt_s390_func_available(KIMD_SHA_256)) - return -EOPNOTSUPP; + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - return crypto_register_shash(&alg); + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + sctx->func = KIMD_SHA_256; + + return 0; +} + +static struct shash_alg sha224_alg = { + .digestsize = SHA224_DIGEST_SIZE, + .init = sha224_init, + .update = s390_sha_update, + .final = s390_sha_final, + .export = sha256_export, + .import = sha256_import, + .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name= "sha224-s390", + .cra_priority = CRYPT_S390_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha256_s390_init(void) +{ + int ret; + + if (!crypt_s390_func_available(KIMD_SHA_256, CRYPT_S390_MSA)) + return -EOPNOTSUPP; + ret = crypto_register_shash(&sha256_alg); + if (ret < 0) + goto out; + ret = crypto_register_shash(&sha224_alg); + if (ret < 0) + crypto_unregister_shash(&sha256_alg); +out: + return ret; } static void __exit sha256_s390_fini(void) { - crypto_unregister_shash(&alg); + crypto_unregister_shash(&sha224_alg); + crypto_unregister_shash(&sha256_alg); } module_init(sha256_s390_init); module_exit(sha256_s390_fini); MODULE_ALIAS("sha256"); +MODULE_ALIAS("sha224"); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm"); +MODULE_DESCRIPTION("SHA256 and SHA224 Secure Hash Algorithm"); diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 4bf73d0dc525..32a81383b69c 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -132,7 +132,7 @@ static int __init init(void) { int ret; - if (!crypt_s390_func_available(KIMD_SHA_512)) + if (!crypt_s390_func_available(KIMD_SHA_512, CRYPT_S390_MSA)) return -EOPNOTSUPP; if ((ret = crypto_register_shash(&sha512_alg)) < 0) goto out; diff --git a/arch/s390/hypfs/hypfs.h b/arch/s390/hypfs/hypfs.h index 80c1526f2af3..d9df5a060a83 100644 --- a/arch/s390/hypfs/hypfs.h +++ b/arch/s390/hypfs/hypfs.h @@ -47,7 +47,7 @@ struct hypfs_dbfs_data { void *buf; void *buf_free_ptr; size_t size; - struct hypfs_dbfs_file *dbfs_file;; + struct hypfs_dbfs_file *dbfs_file; struct kref kref; }; diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index d9db13810d15..8517d2ae3b5c 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -93,7 +93,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return old; } -static inline int atomic_add_unless(atomic_t *v, int a, int u) +static inline int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; c = atomic_read(v); @@ -105,10 +105,9 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) break; c = old; } - return c != u; + return c; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #undef __CS_LOOP @@ -332,6 +331,4 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #define smp_mb__before_atomic_inc() smp_mb() #define smp_mb__after_atomic_inc() smp_mb() -#include <asm-generic/atomic-long.h> - #endif /* __ARCH_S390_ATOMIC__ */ diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index e1c8f3a49884..e5beb490959b 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -621,6 +621,7 @@ static inline unsigned long find_first_zero_bit(const unsigned long *addr, bits = __ffz_word(bytes*8, __load_ulong_be(addr, bytes)); return (bits < size) ? bits : size; } +#define find_first_zero_bit find_first_zero_bit /** * find_first_bit - find the first set bit in a memory region @@ -641,6 +642,7 @@ static inline unsigned long find_first_bit(const unsigned long * addr, bits = __ffs_word(bytes*8, __load_ulong_be(addr, bytes)); return (bits < size) ? bits : size; } +#define find_first_bit find_first_bit /** * find_next_zero_bit - find the first zero bit in a memory region @@ -677,6 +679,7 @@ static inline int find_next_zero_bit (const unsigned long * addr, } return offset + find_first_zero_bit(p, size); } +#define find_next_zero_bit find_next_zero_bit /** * find_next_bit - find the first set bit in a memory region @@ -713,6 +716,7 @@ static inline int find_next_bit (const unsigned long * addr, } return offset + find_first_bit(p, size); } +#define find_next_bit find_next_bit /* * Every architecture must define this function. It's the fastest @@ -742,41 +746,6 @@ static inline int sched_find_first_bit(unsigned long *b) * 23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24 */ -static inline void __set_bit_le(unsigned long nr, void *addr) -{ - __set_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline void __clear_bit_le(unsigned long nr, void *addr) -{ - __clear_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline int __test_and_set_bit_le(unsigned long nr, void *addr) -{ - return __test_and_set_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline int test_and_set_bit_le(unsigned long nr, void *addr) -{ - return test_and_set_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline int __test_and_clear_bit_le(unsigned long nr, void *addr) -{ - return __test_and_clear_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline int test_and_clear_bit_le(unsigned long nr, void *addr) -{ - return test_and_clear_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - -static inline int test_bit_le(unsigned long nr, const void *addr) -{ - return test_bit(nr ^ (__BITOPS_WORDSIZE - 8), addr); -} - static inline int find_first_zero_bit_le(void *vaddr, unsigned int size) { unsigned long bytes, bits; @@ -787,6 +756,7 @@ static inline int find_first_zero_bit_le(void *vaddr, unsigned int size) bits = __ffz_word(bytes*8, __load_ulong_le(vaddr, bytes)); return (bits < size) ? bits : size; } +#define find_first_zero_bit_le find_first_zero_bit_le static inline int find_next_zero_bit_le(void *vaddr, unsigned long size, unsigned long offset) @@ -816,6 +786,7 @@ static inline int find_next_zero_bit_le(void *vaddr, unsigned long size, } return offset + find_first_zero_bit_le(p, size); } +#define find_next_zero_bit_le find_next_zero_bit_le static inline unsigned long find_first_bit_le(void *vaddr, unsigned long size) { @@ -827,6 +798,7 @@ static inline unsigned long find_first_bit_le(void *vaddr, unsigned long size) bits = __ffs_word(bytes*8, __load_ulong_le(vaddr, bytes)); return (bits < size) ? bits : size; } +#define find_first_bit_le find_first_bit_le static inline int find_next_bit_le(void *vaddr, unsigned long size, unsigned long offset) @@ -856,11 +828,11 @@ static inline int find_next_bit_le(void *vaddr, unsigned long size, } return offset + find_first_bit_le(p, size); } +#define find_next_bit_le find_next_bit_le + +#include <asm-generic/bitops/le.h> -#define ext2_set_bit_atomic(lock, nr, addr) \ - test_and_set_bit_le(nr, addr) -#define ext2_clear_bit_atomic(lock, nr, addr) \ - test_and_clear_bit_le(nr, addr) +#include <asm-generic/bitops/ext2-atomic-setbit.h> #endif /* __KERNEL__ */ diff --git a/arch/s390/include/asm/cacheflush.h b/arch/s390/include/asm/cacheflush.h index 43a5c78046db..3e20383d0921 100644 --- a/arch/s390/include/asm/cacheflush.h +++ b/arch/s390/include/asm/cacheflush.h @@ -11,5 +11,6 @@ void kernel_map_pages(struct page *page, int numpages, int enable); int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); +int set_memory_x(unsigned long addr, int numpages); #endif /* _S390_CACHEFLUSH_H */ diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 7488e52efa97..81d7908416cf 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -167,7 +167,6 @@ static inline unsigned long __cmpxchg(void *ptr, unsigned long old, #ifdef CONFIG_64BIT #define cmpxchg64(ptr, o, n) \ ({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg((ptr), (o), (n)); \ }) #else /* CONFIG_64BIT */ diff --git a/arch/s390/include/asm/delay.h b/arch/s390/include/asm/delay.h index 8a096b83f51f..0e3b35f96be1 100644 --- a/arch/s390/include/asm/delay.h +++ b/arch/s390/include/asm/delay.h @@ -14,10 +14,12 @@ #ifndef _S390_DELAY_H #define _S390_DELAY_H -extern void __udelay(unsigned long long usecs); -extern void udelay_simple(unsigned long long usecs); -extern void __delay(unsigned long loops); +void __ndelay(unsigned long long nsecs); +void __udelay(unsigned long long usecs); +void udelay_simple(unsigned long long usecs); +void __delay(unsigned long loops); +#define ndelay(n) __ndelay((unsigned long long) (n)) #define udelay(n) __udelay((unsigned long long) (n)) #define mdelay(n) __udelay((unsigned long long) (n) * 1000) diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 72b2e2f2d32d..7e91c58072e2 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -9,9 +9,22 @@ #define _ASM_S390_DIAG_H /* - * Diagnose 10: Release pages + * Diagnose 10: Release page range */ -extern void diag10(unsigned long addr); +static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn) +{ + unsigned long start_addr, end_addr; + + start_addr = start_pfn << PAGE_SHIFT; + end_addr = (start_pfn + num_pfn - 1) << PAGE_SHIFT; + + asm volatile( + "0: diag %0,%1,0x10\n" + "1:\n" + EX_TABLE(0b, 1b) + EX_TABLE(1b, 1b) + : : "a" (start_addr), "a" (end_addr)); +} /* * Diagnose 14: Input spool file manipulation diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index 10c029cfcc7d..64b61bf72e93 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -196,18 +196,6 @@ do { \ } while (0) #endif /* __s390x__ */ -/* - * An executable for which elf_read_implies_exec() returns TRUE will - * have the READ_IMPLIES_EXEC personality flag set automatically. - */ -#define elf_read_implies_exec(ex, executable_stack) \ -({ \ - if (current->mm->context.noexec && \ - executable_stack != EXSTACK_DISABLE_X) \ - disable_noexec(current->mm, current); \ - current->mm->context.noexec == 0; \ -}) - #define STACK_RND_MASK 0x7ffUL #define ARCH_DLINFO \ diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 3c29be4836ed..b7931faaef6d 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -11,15 +11,13 @@ struct dyn_arch_ftrace { }; #ifdef CONFIG_64BIT #define MCOUNT_INSN_SIZE 12 -#define MCOUNT_OFFSET 8 #else #define MCOUNT_INSN_SIZE 20 -#define MCOUNT_OFFSET 4 #endif static inline unsigned long ftrace_call_adjust(unsigned long addr) { - return addr - MCOUNT_OFFSET; + return addr; } #endif /* __ASSEMBLY__ */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index b56403c2df28..799ed0f1643d 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -111,21 +111,10 @@ static inline void huge_ptep_invalidate(struct mm_struct *mm, { pmd_t *pmdp = (pmd_t *) ptep; - if (!MACHINE_HAS_IDTE) { - __pmd_csp(pmdp); - if (mm->context.noexec) { - pmdp = get_shadow_table(pmdp); - __pmd_csp(pmdp); - } - return; - } - - __pmd_idte(address, pmdp); - if (mm->context.noexec) { - pmdp = get_shadow_table(pmdp); + if (MACHINE_HAS_IDTE) __pmd_idte(address, pmdp); - } - return; + else + __pmd_csp(pmdp); } #define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ diff --git a/arch/s390/include/asm/ipl.h b/arch/s390/include/asm/ipl.h index 5e95d95450b3..97cc4403fabf 100644 --- a/arch/s390/include/asm/ipl.h +++ b/arch/s390/include/asm/ipl.h @@ -167,5 +167,6 @@ enum diag308_rc { }; extern int diag308(unsigned long subcode, void *addr); +extern void diag308_reset(void); #endif /* _ASM_S390_IPL_H */ diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index db14a311f1d2..ba7b01c726a3 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -2,6 +2,7 @@ #define _ASM_IRQ_H #include <linux/hardirq.h> +#include <linux/types.h> enum interruption_class { EXTERNAL_INTERRUPT, @@ -15,6 +16,7 @@ enum interruption_class { EXTINT_VRT, EXTINT_SCP, EXTINT_IUC, + EXTINT_CPM, IOINT_QAI, IOINT_QDI, IOINT_DAS, @@ -30,4 +32,11 @@ enum interruption_class { NR_IRQS, }; +typedef void (*ext_int_handler_t)(unsigned int, unsigned int, unsigned long); + +int register_external_interrupt(u16 code, ext_int_handler_t handler); +int unregister_external_interrupt(u16 code, ext_int_handler_t handler); +void service_subclass_irq_register(void); +void service_subclass_irq_unregister(void); + #endif /* _ASM_IRQ_H */ diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index 865d6d891ace..38fdf451febb 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -29,42 +29,42 @@ }) /* set system mask. */ -static inline void __arch_local_irq_ssm(unsigned long flags) +static inline notrace void __arch_local_irq_ssm(unsigned long flags) { asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -static inline unsigned long arch_local_save_flags(void) +static inline notrace unsigned long arch_local_save_flags(void) { return __arch_local_irq_stosm(0x00); } -static inline unsigned long arch_local_irq_save(void) +static inline notrace unsigned long arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } -static inline void arch_local_irq_disable(void) +static inline notrace void arch_local_irq_disable(void) { arch_local_irq_save(); } -static inline void arch_local_irq_enable(void) +static inline notrace void arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } -static inline void arch_local_irq_restore(unsigned long flags) +static inline notrace void arch_local_irq_restore(unsigned long flags) { __arch_local_irq_ssm(flags); } -static inline bool arch_irqs_disabled_flags(unsigned long flags) +static inline notrace bool arch_irqs_disabled_flags(unsigned long flags) { return !(flags & (3UL << (BITS_PER_LONG - 8))); } -static inline bool arch_irqs_disabled(void) +static inline notrace bool arch_irqs_disabled(void) { return arch_irqs_disabled_flags(arch_local_save_flags()); } diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h new file mode 100644 index 000000000000..95a6cf2b5b67 --- /dev/null +++ b/arch/s390/include/asm/jump_label.h @@ -0,0 +1,37 @@ +#ifndef _ASM_S390_JUMP_LABEL_H +#define _ASM_S390_JUMP_LABEL_H + +#include <linux/types.h> + +#define JUMP_LABEL_NOP_SIZE 6 + +#ifdef CONFIG_64BIT +#define ASM_PTR ".quad" +#define ASM_ALIGN ".balign 8" +#else +#define ASM_PTR ".long" +#define ASM_ALIGN ".balign 4" +#endif + +static __always_inline bool arch_static_branch(struct jump_label_key *key) +{ + asm goto("0: brcl 0,0\n" + ".pushsection __jump_table, \"aw\"\n" + ASM_ALIGN "\n" + ASM_PTR " 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (key) : : label); + return false; +label: + return true; +} + +typedef unsigned long jump_label_t; + +struct jump_entry { + jump_label_t code; + jump_label_t target; + jump_label_t key; +}; + +#endif diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index cef7dbf69dfc..00ff00dfb24c 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -93,9 +93,7 @@ struct kvm_s390_sie_block { __u32 scaol; /* 0x0064 */ __u8 reserved68[4]; /* 0x0068 */ __u32 todpr; /* 0x006c */ - __u8 reserved70[16]; /* 0x0070 */ - __u64 gmsor; /* 0x0080 */ - __u64 gmslm; /* 0x0088 */ + __u8 reserved70[32]; /* 0x0070 */ psw_t gpsw; /* 0x0090 */ __u64 gg14; /* 0x00a0 */ __u64 gg15; /* 0x00a8 */ @@ -138,6 +136,7 @@ struct kvm_vcpu_stat { u32 instruction_chsc; u32 instruction_stsi; u32 instruction_stfl; + u32 instruction_tprot; u32 instruction_sigp_sense; u32 instruction_sigp_emergency; u32 instruction_sigp_stop; @@ -175,6 +174,10 @@ struct kvm_s390_prefix_info { __u32 address; }; +struct kvm_s390_emerg_info { + __u16 code; +}; + struct kvm_s390_interrupt_info { struct list_head list; u64 type; @@ -182,6 +185,7 @@ struct kvm_s390_interrupt_info { struct kvm_s390_io_info io; struct kvm_s390_ext_info ext; struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; struct kvm_s390_prefix_info prefix; }; }; @@ -226,6 +230,7 @@ struct kvm_vcpu_arch { struct cpuid cpu_id; u64 stidp_data; }; + struct gmap *gmap; }; struct kvm_vm_stat { @@ -236,6 +241,7 @@ struct kvm_arch{ struct sca_block *sca; debug_info_t *dbf; struct kvm_s390_float_interrupt float_int; + struct gmap *gmap; }; extern int sie64a(struct kvm_s390_sie_block *, unsigned long *); diff --git a/arch/s390/include/asm/linkage.h b/arch/s390/include/asm/linkage.h index 291c2d01c44f..fc8a8284778e 100644 --- a/arch/s390/include/asm/linkage.h +++ b/arch/s390/include/asm/linkage.h @@ -1,6 +1,9 @@ #ifndef __ASM_LINKAGE_H #define __ASM_LINKAGE_H -/* Nothing to see here... */ +#include <linux/stringify.h> + +#define __ALIGN .align 4, 0x07 +#define __ALIGN_STR __stringify(__ALIGN) #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 65e172f8209d..e85c911aabf0 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -18,6 +18,7 @@ void system_call(void); void pgm_check_handler(void); void mcck_int_handler(void); void io_int_handler(void); +void psw_restart_int_handler(void); #ifdef CONFIG_32BIT @@ -124,7 +125,7 @@ struct _lowcore { /* Address space pointer. */ __u32 kernel_asce; /* 0x02ac */ __u32 user_asce; /* 0x02b0 */ - __u32 user_exec_asce; /* 0x02b4 */ + __u32 current_pid; /* 0x02b4 */ /* SMP info area */ __u32 cpu_nr; /* 0x02b8 */ @@ -150,7 +151,10 @@ struct _lowcore { */ __u32 ipib; /* 0x0e00 */ __u32 ipib_checksum; /* 0x0e04 */ - __u8 pad_0x0e08[0x0f00-0x0e08]; /* 0x0e08 */ + + /* 64 bit save area */ + __u64 save_area_64; /* 0x0e08 */ + __u8 pad_0x0e10[0x0f00-0x0e10]; /* 0x0e10 */ /* Extended facility list */ __u64 stfle_fac_list[32]; /* 0x0f00 */ @@ -255,7 +259,7 @@ struct _lowcore { /* Address space pointer. */ __u64 kernel_asce; /* 0x0310 */ __u64 user_asce; /* 0x0318 */ - __u64 user_exec_asce; /* 0x0320 */ + __u64 current_pid; /* 0x0320 */ /* SMP info area */ __u32 cpu_nr; /* 0x0328 */ @@ -268,7 +272,7 @@ struct _lowcore { __u64 vdso_per_cpu_data; /* 0x0358 */ __u64 machine_flags; /* 0x0360 */ __u64 ftrace_func; /* 0x0368 */ - __u64 sie_hook; /* 0x0370 */ + __u64 gmap; /* 0x0370 */ __u64 cmf_hpp; /* 0x0378 */ /* Interrupt response block. */ @@ -286,7 +290,10 @@ struct _lowcore { */ __u64 ipib; /* 0x0e00 */ __u32 ipib_checksum; /* 0x0e08 */ - __u8 pad_0x0e0c[0x0f00-0x0e0c]; /* 0x0e0c */ + + /* 64 bit save area */ + __u64 save_area_64; /* 0x0e0c */ + __u8 pad_0x0e14[0x0f00-0x0e14]; /* 0x0e14 */ /* Extended facility list */ __u64 stfle_fac_list[32]; /* 0x0f00 */ diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 78522cdefdd4..4506791adcd5 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -5,19 +5,20 @@ typedef struct { atomic_t attach_count; unsigned int flush_mm; spinlock_t list_lock; - struct list_head crst_list; struct list_head pgtable_list; + struct list_head gmap_list; unsigned long asce_bits; unsigned long asce_limit; unsigned long vdso_base; - int noexec; - int has_pgste; /* The mmu context has extended page tables */ - int alloc_pgste; /* cloned contexts will have extended page tables */ + /* Cloned contexts will be created with extended page tables. */ + unsigned int alloc_pgste:1; + /* The mmu context has extended page tables. */ + unsigned int has_pgste:1; } mm_context_t; #define INIT_MM_CONTEXT(name) \ .context.list_lock = __SPIN_LOCK_UNLOCKED(name.context.list_lock), \ - .context.crst_list = LIST_HEAD_INIT(name.context.crst_list), \ - .context.pgtable_list = LIST_HEAD_INIT(name.context.pgtable_list), + .context.pgtable_list = LIST_HEAD_INIT(name.context.pgtable_list), \ + .context.gmap_list = LIST_HEAD_INIT(name.context.gmap_list), #endif diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index a6f0e7cc9cde..5682f160ff82 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -23,7 +23,7 @@ static inline int init_new_context(struct task_struct *tsk, #ifdef CONFIG_64BIT mm->context.asce_bits |= _ASCE_TYPE_REGION3; #endif - if (current->mm->context.alloc_pgste) { + if (current->mm && current->mm->context.alloc_pgste) { /* * alloc_pgste indicates, that any NEW context will be created * with extended page tables. The old context is unchanged. The @@ -35,11 +35,9 @@ static inline int init_new_context(struct task_struct *tsk, * and if has_pgste is set, it will create extended page * tables. */ - mm->context.noexec = 0; mm->context.has_pgste = 1; mm->context.alloc_pgste = 1; } else { - mm->context.noexec = (user_mode == SECONDARY_SPACE_MODE); mm->context.has_pgste = 0; mm->context.alloc_pgste = 0; } @@ -63,10 +61,8 @@ static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk) S390_lowcore.user_asce = mm->context.asce_bits | __pa(pgd); if (user_mode != HOME_SPACE_MODE) { /* Load primary space page table origin. */ - pgd = mm->context.noexec ? get_shadow_table(pgd) : pgd; - S390_lowcore.user_exec_asce = mm->context.asce_bits | __pa(pgd); asm volatile(LCTL_OPCODE" 1,1,%0\n" - : : "m" (S390_lowcore.user_exec_asce) ); + : : "m" (S390_lowcore.user_asce) ); } else /* Load home space page table origin. */ asm volatile(LCTL_OPCODE" 13,13,%0" diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 3c987e9ec8d6..accb372ddc7e 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -90,6 +90,7 @@ static inline void copy_page(void *to, void *from) */ typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct { unsigned long pgste; } pgste_t; typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pmd; } pmd_t; typedef struct { unsigned long pud; } pud_t; @@ -97,18 +98,21 @@ typedef struct { unsigned long pgd; } pgd_t; typedef pte_t *pgtable_t; #define pgprot_val(x) ((x).pgprot) +#define pgste_val(x) ((x).pgste) #define pte_val(x) ((x).pte) #define pmd_val(x) ((x).pmd) #define pud_val(x) ((x).pud) #define pgd_val(x) ((x).pgd) +#define __pgste(x) ((pgste_t) { (x) } ) #define __pte(x) ((pte_t) { (x) } ) #define __pmd(x) ((pmd_t) { (x) } ) +#define __pud(x) ((pud_t) { (x) } ) #define __pgd(x) ((pgd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) -static inline void -page_set_storage_key(unsigned long addr, unsigned int skey, int mapped) +static inline void page_set_storage_key(unsigned long addr, + unsigned char skey, int mapped) { if (!mapped) asm volatile(".insn rrf,0xb22b0000,%0,%1,8,0" @@ -117,15 +121,59 @@ page_set_storage_key(unsigned long addr, unsigned int skey, int mapped) asm volatile("sske %0,%1" : : "d" (skey), "a" (addr)); } -static inline unsigned int -page_get_storage_key(unsigned long addr) +static inline unsigned char page_get_storage_key(unsigned long addr) { - unsigned int skey; + unsigned char skey; - asm volatile("iske %0,%1" : "=d" (skey) : "a" (addr), "0" (0)); + asm volatile("iske %0,%1" : "=d" (skey) : "a" (addr)); return skey; } +static inline int page_reset_referenced(unsigned long addr) +{ + unsigned int ipm; + + asm volatile( + " rrbe 0,%1\n" + " ipm %0\n" + : "=d" (ipm) : "a" (addr) : "cc"); + return !!(ipm & 0x20000000); +} + +/* Bits int the storage key */ +#define _PAGE_CHANGED 0x02 /* HW changed bit */ +#define _PAGE_REFERENCED 0x04 /* HW referenced bit */ +#define _PAGE_FP_BIT 0x08 /* HW fetch protection bit */ +#define _PAGE_ACC_BITS 0xf0 /* HW access control bits */ + +/* + * Test and clear dirty bit in storage key. + * We can't clear the changed bit atomically. This is a potential + * race against modification of the referenced bit. This function + * should therefore only be called if it is not mapped in any + * address space. + */ +#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_DIRTY +static inline int page_test_and_clear_dirty(unsigned long pfn, int mapped) +{ + unsigned char skey; + + skey = page_get_storage_key(pfn << PAGE_SHIFT); + if (!(skey & _PAGE_CHANGED)) + return 0; + page_set_storage_key(pfn << PAGE_SHIFT, skey & ~_PAGE_CHANGED, mapped); + return 1; +} + +/* + * Test and clear referenced bit in storage key. + */ +#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG +static inline int page_test_and_clear_young(unsigned long pfn) +{ + return page_reset_referenced(pfn << PAGE_SHIFT); +} + struct page; void arch_free_page(struct page *page, int order); void arch_alloc_page(struct page *page, int order); diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index f7ad8719d02d..5325c89a5843 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -1,6 +1,9 @@ #ifndef __ARCH_S390_PERCPU__ #define __ARCH_S390_PERCPU__ +#include <linux/preempt.h> +#include <asm/cmpxchg.h> + /* * s390 uses its own implementation for per cpu data, the offset of * the cpu local data area is cached in the cpu's lowcore memory. @@ -16,6 +19,71 @@ #define ARCH_NEEDS_WEAK_PER_CPU #endif +#define arch_irqsafe_cpu_to_op(pcp, val, op) \ +do { \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ old__, new__, prev__; \ + pcp_op_T__ *ptr__; \ + preempt_disable(); \ + ptr__ = __this_cpu_ptr(&(pcp)); \ + prev__ = *ptr__; \ + do { \ + old__ = prev__; \ + new__ = old__ op (val); \ + switch (sizeof(*ptr__)) { \ + case 8: \ + prev__ = cmpxchg64(ptr__, old__, new__); \ + break; \ + default: \ + prev__ = cmpxchg(ptr__, old__, new__); \ + } \ + } while (prev__ != old__); \ + preempt_enable(); \ +} while (0) + +#define irqsafe_cpu_add_1(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, +) +#define irqsafe_cpu_add_2(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, +) +#define irqsafe_cpu_add_4(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, +) +#define irqsafe_cpu_add_8(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, +) + +#define irqsafe_cpu_and_1(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, &) +#define irqsafe_cpu_and_2(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, &) +#define irqsafe_cpu_and_4(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, &) +#define irqsafe_cpu_and_8(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, &) + +#define irqsafe_cpu_or_1(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, |) +#define irqsafe_cpu_or_2(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, |) +#define irqsafe_cpu_or_4(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, |) +#define irqsafe_cpu_or_8(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, |) + +#define irqsafe_cpu_xor_1(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, ^) +#define irqsafe_cpu_xor_2(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, ^) +#define irqsafe_cpu_xor_4(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, ^) +#define irqsafe_cpu_xor_8(pcp, val) arch_irqsafe_cpu_to_op(pcp, val, ^) + +#define arch_irqsafe_cpu_cmpxchg(pcp, oval, nval) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ ret__; \ + pcp_op_T__ *ptr__; \ + preempt_disable(); \ + ptr__ = __this_cpu_ptr(&(pcp)); \ + switch (sizeof(*ptr__)) { \ + case 8: \ + ret__ = cmpxchg64(ptr__, oval, nval); \ + break; \ + default: \ + ret__ = cmpxchg(ptr__, oval, nval); \ + } \ + preempt_enable(); \ + ret__; \ +}) + +#define irqsafe_cpu_cmpxchg_1(pcp, oval, nval) arch_irqsafe_cpu_cmpxchg(pcp, oval, nval) +#define irqsafe_cpu_cmpxchg_2(pcp, oval, nval) arch_irqsafe_cpu_cmpxchg(pcp, oval, nval) +#define irqsafe_cpu_cmpxchg_4(pcp, oval, nval) arch_irqsafe_cpu_cmpxchg(pcp, oval, nval) +#define irqsafe_cpu_cmpxchg_8(pcp, oval, nval) arch_irqsafe_cpu_cmpxchg(pcp, oval, nval) + #include <asm-generic/percpu.h> #endif /* __ARCH_S390_PERCPU__ */ diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index 082eb4e50e8b..8eef9b5b3cf4 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -17,16 +17,15 @@ #include <linux/gfp.h> #include <linux/mm.h> -#define check_pgt_cache() do {} while (0) - -unsigned long *crst_table_alloc(struct mm_struct *, int); +unsigned long *crst_table_alloc(struct mm_struct *); void crst_table_free(struct mm_struct *, unsigned long *); -void crst_table_free_rcu(struct mm_struct *, unsigned long *); -unsigned long *page_table_alloc(struct mm_struct *); +unsigned long *page_table_alloc(struct mm_struct *, unsigned long); void page_table_free(struct mm_struct *, unsigned long *); -void page_table_free_rcu(struct mm_struct *, unsigned long *); -void disable_noexec(struct mm_struct *, struct task_struct *); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE +void page_table_free_rcu(struct mmu_gather *, unsigned long *); +void __tlb_remove_table(void *_table); +#endif static inline void clear_table(unsigned long *s, unsigned long val, size_t n) { @@ -50,9 +49,6 @@ static inline void clear_table(unsigned long *s, unsigned long val, size_t n) static inline void crst_table_init(unsigned long *crst, unsigned long entry) { clear_table(crst, entry, sizeof(unsigned long)*2048); - crst = get_shadow_table(crst); - if (crst) - clear_table(crst, entry, sizeof(unsigned long)*2048); } #ifndef __s390x__ @@ -69,10 +65,7 @@ static inline unsigned long pgd_entry_type(struct mm_struct *mm) #define pmd_free(mm, x) do { } while (0) #define pgd_populate(mm, pgd, pud) BUG() -#define pgd_populate_kernel(mm, pgd, pud) BUG() - #define pud_populate(mm, pud, pmd) BUG() -#define pud_populate_kernel(mm, pud, pmd) BUG() #else /* __s390x__ */ @@ -90,7 +83,7 @@ void crst_table_downgrade(struct mm_struct *, unsigned long limit); static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) { - unsigned long *table = crst_table_alloc(mm, mm->context.noexec); + unsigned long *table = crst_table_alloc(mm); if (table) crst_table_init(table, _REGION3_ENTRY_EMPTY); return (pud_t *) table; @@ -99,43 +92,21 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr) { - unsigned long *table = crst_table_alloc(mm, mm->context.noexec); + unsigned long *table = crst_table_alloc(mm); if (table) crst_table_init(table, _SEGMENT_ENTRY_EMPTY); return (pmd_t *) table; } #define pmd_free(mm, pmd) crst_table_free(mm, (unsigned long *) pmd) -static inline void pgd_populate_kernel(struct mm_struct *mm, - pgd_t *pgd, pud_t *pud) -{ - pgd_val(*pgd) = _REGION2_ENTRY | __pa(pud); -} - static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) { - pgd_populate_kernel(mm, pgd, pud); - if (mm->context.noexec) { - pgd = get_shadow_table(pgd); - pud = get_shadow_table(pud); - pgd_populate_kernel(mm, pgd, pud); - } -} - -static inline void pud_populate_kernel(struct mm_struct *mm, - pud_t *pud, pmd_t *pmd) -{ - pud_val(*pud) = _REGION3_ENTRY | __pa(pmd); + pgd_val(*pgd) = _REGION2_ENTRY | __pa(pud); } static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { - pud_populate_kernel(mm, pud, pmd); - if (mm->context.noexec) { - pud = get_shadow_table(pud); - pmd = get_shadow_table(pmd); - pud_populate_kernel(mm, pud, pmd); - } + pud_val(*pud) = _REGION3_ENTRY | __pa(pmd); } #endif /* __s390x__ */ @@ -143,37 +114,28 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline pgd_t *pgd_alloc(struct mm_struct *mm) { spin_lock_init(&mm->context.list_lock); - INIT_LIST_HEAD(&mm->context.crst_list); INIT_LIST_HEAD(&mm->context.pgtable_list); - return (pgd_t *) - crst_table_alloc(mm, user_mode == SECONDARY_SPACE_MODE); + INIT_LIST_HEAD(&mm->context.gmap_list); + return (pgd_t *) crst_table_alloc(mm); } #define pgd_free(mm, pgd) crst_table_free(mm, (unsigned long *) pgd) -static inline void pmd_populate_kernel(struct mm_struct *mm, - pmd_t *pmd, pte_t *pte) -{ - pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte); -} - static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) { - pmd_populate_kernel(mm, pmd, pte); - if (mm->context.noexec) { - pmd = get_shadow_table(pmd); - pmd_populate_kernel(mm, pmd, pte + PTRS_PER_PTE); - } + pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte); } +#define pmd_populate_kernel(mm, pmd, pte) pmd_populate(mm, pmd, pte) + #define pmd_pgtable(pmd) \ (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE) /* * page table entry allocation/free routines. */ -#define pte_alloc_one_kernel(mm, vmaddr) ((pte_t *) page_table_alloc(mm)) -#define pte_alloc_one(mm, vmaddr) ((pte_t *) page_table_alloc(mm)) +#define pte_alloc_one_kernel(mm, vmaddr) ((pte_t *) page_table_alloc(mm, vmaddr)) +#define pte_alloc_one(mm, vmaddr) ((pte_t *) page_table_alloc(mm, vmaddr)) #define pte_free_kernel(mm, pte) page_table_free(mm, (unsigned long *) pte) #define pte_free(mm, pte) page_table_free(mm, (unsigned long *) pte) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 02ace3491c51..519eb5f187ef 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -31,9 +31,8 @@ #ifndef __ASSEMBLY__ #include <linux/sched.h> #include <linux/mm_types.h> -#include <asm/bitops.h> #include <asm/bug.h> -#include <asm/processor.h> +#include <asm/page.h> extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096))); extern void paging_init(void); @@ -243,11 +242,13 @@ extern unsigned long VMALLOC_START; /* Software bits in the page table entry */ #define _PAGE_SWT 0x001 /* SW pte type bit t */ #define _PAGE_SWX 0x002 /* SW pte type bit x */ -#define _PAGE_SPECIAL 0x004 /* SW associated with special page */ +#define _PAGE_SWC 0x004 /* SW pte changed bit (for KVM) */ +#define _PAGE_SWR 0x008 /* SW pte referenced bit (for KVM) */ +#define _PAGE_SPECIAL 0x010 /* SW associated with special page */ #define __HAVE_ARCH_PTE_SPECIAL /* Set of bits not changed in pte_modify */ -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL) +#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_SWC | _PAGE_SWR) /* Six different types of pages. */ #define _PAGE_TYPE_EMPTY 0x400 @@ -256,8 +257,6 @@ extern unsigned long VMALLOC_START; #define _PAGE_TYPE_FILE 0x601 /* bit 0x002 is used for offset !! */ #define _PAGE_TYPE_RO 0x200 #define _PAGE_TYPE_RW 0x000 -#define _PAGE_TYPE_EX_RO 0x202 -#define _PAGE_TYPE_EX_RW 0x002 /* * Only four types for huge pages, using the invalid bit and protection bit @@ -287,8 +286,6 @@ extern unsigned long VMALLOC_START; * _PAGE_TYPE_FILE 11?1 -> 11?1 * _PAGE_TYPE_RO 0100 -> 1100 * _PAGE_TYPE_RW 0000 -> 1000 - * _PAGE_TYPE_EX_RO 0110 -> 1110 - * _PAGE_TYPE_EX_RW 0010 -> 1010 * * pte_none is true for bits combinations 1000, 1010, 1100, 1110 * pte_present is true for bits combinations 0000, 0010, 0100, 0110, 1001 @@ -296,16 +293,6 @@ extern unsigned long VMALLOC_START; * swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid. */ -/* Page status table bits for virtualization */ -#define RCP_PCL_BIT 55 -#define RCP_HR_BIT 54 -#define RCP_HC_BIT 53 -#define RCP_GR_BIT 50 -#define RCP_GC_BIT 49 - -/* User dirty bit for KVM's migration feature */ -#define KVM_UD_BIT 47 - #ifndef __s390x__ /* Bits in the segment table address-space-control-element */ @@ -325,6 +312,19 @@ extern unsigned long VMALLOC_START; #define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL) #define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) +/* Page status table bits for virtualization */ +#define RCP_ACC_BITS 0xf0000000UL +#define RCP_FP_BIT 0x08000000UL +#define RCP_PCL_BIT 0x00800000UL +#define RCP_HR_BIT 0x00400000UL +#define RCP_HC_BIT 0x00200000UL +#define RCP_GR_BIT 0x00040000UL +#define RCP_GC_BIT 0x00020000UL + +/* User dirty / referenced bit for KVM's migration feature */ +#define KVM_UR_BIT 0x00008000UL +#define KVM_UC_BIT 0x00004000UL + #else /* __s390x__ */ /* Bits in the segment/region table address-space-control-element */ @@ -367,6 +367,19 @@ extern unsigned long VMALLOC_START; #define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */ #define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */ +/* Page status table bits for virtualization */ +#define RCP_ACC_BITS 0xf000000000000000UL +#define RCP_FP_BIT 0x0800000000000000UL +#define RCP_PCL_BIT 0x0080000000000000UL +#define RCP_HR_BIT 0x0040000000000000UL +#define RCP_HC_BIT 0x0020000000000000UL +#define RCP_GR_BIT 0x0004000000000000UL +#define RCP_GC_BIT 0x0002000000000000UL + +/* User dirty / referenced bit for KVM's migration feature */ +#define KVM_UR_BIT 0x0000800000000000UL +#define KVM_UC_BIT 0x0000400000000000UL + #endif /* __s390x__ */ /* @@ -377,85 +390,54 @@ extern unsigned long VMALLOC_START; #define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \ _ASCE_ALT_EVENT) -/* Bits int the storage key */ -#define _PAGE_CHANGED 0x02 /* HW changed bit */ -#define _PAGE_REFERENCED 0x04 /* HW referenced bit */ - /* * Page protection definitions. */ #define PAGE_NONE __pgprot(_PAGE_TYPE_NONE) #define PAGE_RO __pgprot(_PAGE_TYPE_RO) #define PAGE_RW __pgprot(_PAGE_TYPE_RW) -#define PAGE_EX_RO __pgprot(_PAGE_TYPE_EX_RO) -#define PAGE_EX_RW __pgprot(_PAGE_TYPE_EX_RW) #define PAGE_KERNEL PAGE_RW #define PAGE_COPY PAGE_RO /* - * Dependent on the EXEC_PROTECT option s390 can do execute protection. - * Write permission always implies read permission. In theory with a - * primary/secondary page table execute only can be implemented but - * it would cost an additional bit in the pte to distinguish all the - * different pte types. To avoid that execute permission currently - * implies read permission as well. + * On s390 the page table entry has an invalid bit and a read-only bit. + * Read permission implies execute permission and write permission + * implies read permission. */ /*xwr*/ #define __P000 PAGE_NONE #define __P001 PAGE_RO #define __P010 PAGE_RO #define __P011 PAGE_RO -#define __P100 PAGE_EX_RO -#define __P101 PAGE_EX_RO -#define __P110 PAGE_EX_RO -#define __P111 PAGE_EX_RO +#define __P100 PAGE_RO +#define __P101 PAGE_RO +#define __P110 PAGE_RO +#define __P111 PAGE_RO #define __S000 PAGE_NONE #define __S001 PAGE_RO #define __S010 PAGE_RW #define __S011 PAGE_RW -#define __S100 PAGE_EX_RO -#define __S101 PAGE_EX_RO -#define __S110 PAGE_EX_RW -#define __S111 PAGE_EX_RW - -#ifndef __s390x__ -# define PxD_SHADOW_SHIFT 1 -#else /* __s390x__ */ -# define PxD_SHADOW_SHIFT 2 -#endif /* __s390x__ */ +#define __S100 PAGE_RO +#define __S101 PAGE_RO +#define __S110 PAGE_RW +#define __S111 PAGE_RW -static inline void *get_shadow_table(void *table) +static inline int mm_exclusive(struct mm_struct *mm) { - unsigned long addr, offset; - struct page *page; - - addr = (unsigned long) table; - offset = addr & ((PAGE_SIZE << PxD_SHADOW_SHIFT) - 1); - page = virt_to_page((void *)(addr ^ offset)); - return (void *)(addr_t)(page->index ? (page->index | offset) : 0UL); + return likely(mm == current->active_mm && + atomic_read(&mm->context.attach_count) <= 1); } -/* - * Certain architectures need to do special things when PTEs - * within a page table are directly modified. Thus, the following - * hook is made available. - */ -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry) +static inline int mm_has_pgste(struct mm_struct *mm) { - *ptep = entry; - if (mm->context.noexec) { - if (!(pte_val(entry) & _PAGE_INVALID) && - (pte_val(entry) & _PAGE_SWX)) - pte_val(entry) |= _PAGE_RO; - else - pte_val(entry) = _PAGE_TYPE_EMPTY; - ptep[PTRS_PER_PTE] = entry; - } +#ifdef CONFIG_PGSTE + if (unlikely(mm->context.has_pgste)) + return 1; +#endif + return 0; } - /* * pgd/pmd/pte query functions */ @@ -568,54 +550,171 @@ static inline int pte_special(pte_t pte) } #define __HAVE_ARCH_PTE_SAME -#define pte_same(a,b) (pte_val(a) == pte_val(b)) +static inline int pte_same(pte_t a, pte_t b) +{ + return pte_val(a) == pte_val(b); +} -static inline void rcp_lock(pte_t *ptep) +static inline pgste_t pgste_get_lock(pte_t *ptep) { + unsigned long new = 0; #ifdef CONFIG_PGSTE - unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE); + unsigned long old; + preempt_disable(); - while (test_and_set_bit(RCP_PCL_BIT, pgste)) - ; + asm( + " lg %0,%2\n" + "0: lgr %1,%0\n" + " nihh %0,0xff7f\n" /* clear RCP_PCL_BIT in old */ + " oihh %1,0x0080\n" /* set RCP_PCL_BIT in new */ + " csg %0,%1,%2\n" + " jl 0b\n" + : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) + : "Q" (ptep[PTRS_PER_PTE]) : "cc"); #endif + return __pgste(new); } -static inline void rcp_unlock(pte_t *ptep) +static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) { #ifdef CONFIG_PGSTE - unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE); - clear_bit(RCP_PCL_BIT, pgste); + asm( + " nihh %1,0xff7f\n" /* clear RCP_PCL_BIT */ + " stg %1,%0\n" + : "=Q" (ptep[PTRS_PER_PTE]) + : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) : "cc"); preempt_enable(); #endif } -/* forward declaration for SetPageUptodate in page-flags.h*/ -static inline void page_clear_dirty(struct page *page, int mapped); -#include <linux/page-flags.h> - -static inline void ptep_rcp_copy(pte_t *ptep) +static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste) { #ifdef CONFIG_PGSTE - struct page *page = virt_to_page(pte_val(*ptep)); - unsigned int skey; - unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE); - - skey = page_get_storage_key(page_to_phys(page)); - if (skey & _PAGE_CHANGED) { - set_bit_simple(RCP_GC_BIT, pgste); - set_bit_simple(KVM_UD_BIT, pgste); - } - if (skey & _PAGE_REFERENCED) - set_bit_simple(RCP_GR_BIT, pgste); - if (test_and_clear_bit_simple(RCP_HC_BIT, pgste)) { - SetPageDirty(page); - set_bit_simple(KVM_UD_BIT, pgste); + unsigned long address, bits; + unsigned char skey; + + address = pte_val(*ptep) & PAGE_MASK; + skey = page_get_storage_key(address); + bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); + /* Clear page changed & referenced bit in the storage key */ + if (bits) { + skey ^= bits; + page_set_storage_key(address, skey, 1); } - if (test_and_clear_bit_simple(RCP_HR_BIT, pgste)) - SetPageReferenced(page); + /* Transfer page changed & referenced bit to guest bits in pgste */ + pgste_val(pgste) |= bits << 48; /* RCP_GR_BIT & RCP_GC_BIT */ + /* Get host changed & referenced bits from pgste */ + bits |= (pgste_val(pgste) & (RCP_HR_BIT | RCP_HC_BIT)) >> 52; + /* Clear host bits in pgste. */ + pgste_val(pgste) &= ~(RCP_HR_BIT | RCP_HC_BIT); + pgste_val(pgste) &= ~(RCP_ACC_BITS | RCP_FP_BIT); + /* Copy page access key and fetch protection bit to pgste */ + pgste_val(pgste) |= + (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; + /* Transfer changed and referenced to kvm user bits */ + pgste_val(pgste) |= bits << 45; /* KVM_UR_BIT & KVM_UC_BIT */ + /* Transfer changed & referenced to pte sofware bits */ + pte_val(*ptep) |= bits << 1; /* _PAGE_SWR & _PAGE_SWC */ +#endif + return pgste; + +} + +static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste) +{ +#ifdef CONFIG_PGSTE + int young; + + young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); + /* Transfer page referenced bit to pte software bit (host view) */ + if (young || (pgste_val(pgste) & RCP_HR_BIT)) + pte_val(*ptep) |= _PAGE_SWR; + /* Clear host referenced bit in pgste. */ + pgste_val(pgste) &= ~RCP_HR_BIT; + /* Transfer page referenced bit to guest bit in pgste */ + pgste_val(pgste) |= (unsigned long) young << 50; /* set RCP_GR_BIT */ +#endif + return pgste; + +} + +static inline void pgste_set_pte(pte_t *ptep, pgste_t pgste) +{ +#ifdef CONFIG_PGSTE + unsigned long address; + unsigned long okey, nkey; + + address = pte_val(*ptep) & PAGE_MASK; + okey = nkey = page_get_storage_key(address); + nkey &= ~(_PAGE_ACC_BITS | _PAGE_FP_BIT); + /* Set page access key and fetch protection bit from pgste */ + nkey |= (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56; + if (okey != nkey) + page_set_storage_key(address, nkey, 1); #endif } +/** + * struct gmap_struct - guest address space + * @mm: pointer to the parent mm_struct + * @table: pointer to the page directory + * @crst_list: list of all crst tables used in the guest address space + */ +struct gmap { + struct list_head list; + struct mm_struct *mm; + unsigned long *table; + struct list_head crst_list; +}; + +/** + * struct gmap_rmap - reverse mapping for segment table entries + * @next: pointer to the next gmap_rmap structure in the list + * @entry: pointer to a segment table entry + */ +struct gmap_rmap { + struct list_head list; + unsigned long *entry; +}; + +/** + * struct gmap_pgtable - gmap information attached to a page table + * @vmaddr: address of the 1MB segment in the process virtual memory + * @mapper: list of segment table entries maping a page table + */ +struct gmap_pgtable { + unsigned long vmaddr; + struct list_head mapper; +}; + +struct gmap *gmap_alloc(struct mm_struct *mm); +void gmap_free(struct gmap *gmap); +void gmap_enable(struct gmap *gmap); +void gmap_disable(struct gmap *gmap); +int gmap_map_segment(struct gmap *gmap, unsigned long from, + unsigned long to, unsigned long length); +int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len); +unsigned long gmap_fault(unsigned long address, struct gmap *); + +/* + * Certain architectures need to do special things when PTEs + * within a page table are directly modified. Thus, the following + * hook is made available. + */ +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry) +{ + pgste_t pgste; + + if (mm_has_pgste(mm)) { + pgste = pgste_get_lock(ptep); + pgste_set_pte(ptep, pgste); + *ptep = entry; + pgste_set_unlock(ptep, pgste); + } else + *ptep = entry; +} + /* * query functions pte_write/pte_dirty/pte_young only work if * pte_present() is true. Undefined behaviour if not.. @@ -627,19 +726,19 @@ static inline int pte_write(pte_t pte) static inline int pte_dirty(pte_t pte) { - /* A pte is neither clean nor dirty on s/390. The dirty bit - * is in the storage key. See page_test_and_clear_dirty for - * details. - */ +#ifdef CONFIG_PGSTE + if (pte_val(pte) & _PAGE_SWC) + return 1; +#endif return 0; } static inline int pte_young(pte_t pte) { - /* A pte is neither young nor old on s/390. The young bit - * is in the storage key. See page_test_and_clear_young for - * details. - */ +#ifdef CONFIG_PGSTE + if (pte_val(pte) & _PAGE_SWR) + return 1; +#endif return 0; } @@ -647,64 +746,30 @@ static inline int pte_young(pte_t pte) * pgd/pmd/pte modification functions */ -#ifndef __s390x__ - -#define pgd_clear(pgd) do { } while (0) -#define pud_clear(pud) do { } while (0) - -#else /* __s390x__ */ - -static inline void pgd_clear_kernel(pgd_t * pgd) +static inline void pgd_clear(pgd_t *pgd) { +#ifdef __s390x__ if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) pgd_val(*pgd) = _REGION2_ENTRY_EMPTY; +#endif } -static inline void pgd_clear(pgd_t * pgd) -{ - pgd_t *shadow = get_shadow_table(pgd); - - pgd_clear_kernel(pgd); - if (shadow) - pgd_clear_kernel(shadow); -} - -static inline void pud_clear_kernel(pud_t *pud) +static inline void pud_clear(pud_t *pud) { +#ifdef __s390x__ if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) pud_val(*pud) = _REGION3_ENTRY_EMPTY; +#endif } -static inline void pud_clear(pud_t *pud) -{ - pud_t *shadow = get_shadow_table(pud); - - pud_clear_kernel(pud); - if (shadow) - pud_clear_kernel(shadow); -} - -#endif /* __s390x__ */ - -static inline void pmd_clear_kernel(pmd_t * pmdp) +static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; } -static inline void pmd_clear(pmd_t *pmd) -{ - pmd_t *shadow = get_shadow_table(pmd); - - pmd_clear_kernel(pmd); - if (shadow) - pmd_clear_kernel(shadow); -} - static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_val(*ptep) = _PAGE_TYPE_EMPTY; - if (mm->context.noexec) - pte_val(ptep[PTRS_PER_PTE]) = _PAGE_TYPE_EMPTY; } /* @@ -734,35 +799,27 @@ static inline pte_t pte_mkwrite(pte_t pte) static inline pte_t pte_mkclean(pte_t pte) { - /* The only user of pte_mkclean is the fork() code. - We must *not* clear the *physical* page dirty bit - just because fork() wants to clear the dirty bit in - *one* of the page's mappings. So we just do nothing. */ +#ifdef CONFIG_PGSTE + pte_val(pte) &= ~_PAGE_SWC; +#endif return pte; } static inline pte_t pte_mkdirty(pte_t pte) { - /* We do not explicitly set the dirty bit because the - * sske instruction is slow. It is faster to let the - * next instruction set the dirty bit. - */ return pte; } static inline pte_t pte_mkold(pte_t pte) { - /* S/390 doesn't keep its dirty/referenced bit in the pte. - * There is no point in clearing the real referenced bit. - */ +#ifdef CONFIG_PGSTE + pte_val(pte) &= ~_PAGE_SWR; +#endif return pte; } static inline pte_t pte_mkyoung(pte_t pte) { - /* S/390 doesn't keep its dirty/referenced bit in the pte. - * There is no point in setting the real referenced bit. - */ return pte; } @@ -800,62 +857,60 @@ static inline pte_t pte_mkhuge(pte_t pte) } #endif -#ifdef CONFIG_PGSTE /* - * Get (and clear) the user dirty bit for a PTE. + * Get (and clear) the user dirty bit for a pte. */ -static inline int kvm_s390_test_and_clear_page_dirty(struct mm_struct *mm, - pte_t *ptep) +static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm, + pte_t *ptep) { - int dirty; - unsigned long *pgste; - struct page *page; - unsigned int skey; - - if (!mm->context.has_pgste) - return -EINVAL; - rcp_lock(ptep); - pgste = (unsigned long *) (ptep + PTRS_PER_PTE); - page = virt_to_page(pte_val(*ptep)); - skey = page_get_storage_key(page_to_phys(page)); - if (skey & _PAGE_CHANGED) { - set_bit_simple(RCP_GC_BIT, pgste); - set_bit_simple(KVM_UD_BIT, pgste); - } - if (test_and_clear_bit_simple(RCP_HC_BIT, pgste)) { - SetPageDirty(page); - set_bit_simple(KVM_UD_BIT, pgste); + pgste_t pgste; + int dirty = 0; + + if (mm_has_pgste(mm)) { + pgste = pgste_get_lock(ptep); + pgste = pgste_update_all(ptep, pgste); + dirty = !!(pgste_val(pgste) & KVM_UC_BIT); + pgste_val(pgste) &= ~KVM_UC_BIT; + pgste_set_unlock(ptep, pgste); + return dirty; } - dirty = test_and_clear_bit_simple(KVM_UD_BIT, pgste); - if (skey & _PAGE_CHANGED) - page_clear_dirty(page, 1); - rcp_unlock(ptep); return dirty; } -#endif + +/* + * Get (and clear) the user referenced bit for a pte. + */ +static inline int ptep_test_and_clear_user_young(struct mm_struct *mm, + pte_t *ptep) +{ + pgste_t pgste; + int young = 0; + + if (mm_has_pgste(mm)) { + pgste = pgste_get_lock(ptep); + pgste = pgste_update_young(ptep, pgste); + young = !!(pgste_val(pgste) & KVM_UR_BIT); + pgste_val(pgste) &= ~KVM_UR_BIT; + pgste_set_unlock(ptep, pgste); + } + return young; +} #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { -#ifdef CONFIG_PGSTE - unsigned long physpage; - int young; - unsigned long *pgste; + pgste_t pgste; + pte_t pte; - if (!vma->vm_mm->context.has_pgste) - return 0; - physpage = pte_val(*ptep) & PAGE_MASK; - pgste = (unsigned long *) (ptep + PTRS_PER_PTE); - - young = ((page_get_storage_key(physpage) & _PAGE_REFERENCED) != 0); - rcp_lock(ptep); - if (young) - set_bit_simple(RCP_GR_BIT, pgste); - young |= test_and_clear_bit_simple(RCP_HR_BIT, pgste); - rcp_unlock(ptep); - return young; -#endif + if (mm_has_pgste(vma->vm_mm)) { + pgste = pgste_get_lock(ptep); + pgste = pgste_update_young(ptep, pgste); + pte = *ptep; + *ptep = pte_mkold(pte); + pgste_set_unlock(ptep, pgste); + return pte_young(pte); + } return 0; } @@ -867,10 +922,7 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, * On s390 reference bits are in storage key and never in TLB * With virtualization we handle the reference bit, without we * we can simply return */ -#ifdef CONFIG_PGSTE return ptep_test_and_clear_young(vma, address, ptep); -#endif - return 0; } static inline void __ptep_ipte(unsigned long address, pte_t *ptep) @@ -890,25 +942,6 @@ static inline void __ptep_ipte(unsigned long address, pte_t *ptep) } } -static inline void ptep_invalidate(struct mm_struct *mm, - unsigned long address, pte_t *ptep) -{ - if (mm->context.has_pgste) { - rcp_lock(ptep); - __ptep_ipte(address, ptep); - ptep_rcp_copy(ptep); - pte_val(*ptep) = _PAGE_TYPE_EMPTY; - rcp_unlock(ptep); - return; - } - __ptep_ipte(address, ptep); - pte_val(*ptep) = _PAGE_TYPE_EMPTY; - if (mm->context.noexec) { - __ptep_ipte(address, ptep + PTRS_PER_PTE); - pte_val(*(ptep + PTRS_PER_PTE)) = _PAGE_TYPE_EMPTY; - } -} - /* * This is hard to understand. ptep_get_and_clear and ptep_clear_flush * both clear the TLB for the unmapped pte. The reason is that @@ -923,24 +956,72 @@ static inline void ptep_invalidate(struct mm_struct *mm, * is a nop. */ #define __HAVE_ARCH_PTEP_GET_AND_CLEAR -#define ptep_get_and_clear(__mm, __address, __ptep) \ -({ \ - pte_t __pte = *(__ptep); \ - (__mm)->context.flush_mm = 1; \ - if (atomic_read(&(__mm)->context.attach_count) > 1 || \ - (__mm) != current->active_mm) \ - ptep_invalidate(__mm, __address, __ptep); \ - else \ - pte_clear((__mm), (__address), (__ptep)); \ - __pte; \ -}) +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + pgste_t pgste; + pte_t pte; + + mm->context.flush_mm = 1; + if (mm_has_pgste(mm)) + pgste = pgste_get_lock(ptep); + + pte = *ptep; + if (!mm_exclusive(mm)) + __ptep_ipte(address, ptep); + pte_val(*ptep) = _PAGE_TYPE_EMPTY; + + if (mm_has_pgste(mm)) { + pgste = pgste_update_all(&pte, pgste); + pgste_set_unlock(ptep, pgste); + } + return pte; +} + +#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte; + + mm->context.flush_mm = 1; + if (mm_has_pgste(mm)) + pgste_get_lock(ptep); + + pte = *ptep; + if (!mm_exclusive(mm)) + __ptep_ipte(address, ptep); + return pte; +} + +static inline void ptep_modify_prot_commit(struct mm_struct *mm, + unsigned long address, + pte_t *ptep, pte_t pte) +{ + *ptep = pte; + if (mm_has_pgste(mm)) + pgste_set_unlock(ptep, *(pgste_t *)(ptep + PTRS_PER_PTE)); +} #define __HAVE_ARCH_PTEP_CLEAR_FLUSH static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { - pte_t pte = *ptep; - ptep_invalidate(vma->vm_mm, address, ptep); + pgste_t pgste; + pte_t pte; + + if (mm_has_pgste(vma->vm_mm)) + pgste = pgste_get_lock(ptep); + + pte = *ptep; + __ptep_ipte(address, ptep); + pte_val(*ptep) = _PAGE_TYPE_EMPTY; + + if (mm_has_pgste(vma->vm_mm)) { + pgste = pgste_update_all(&pte, pgste); + pgste_set_unlock(ptep, pgste); + } return pte; } @@ -953,76 +1034,67 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, */ #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, - unsigned long addr, + unsigned long address, pte_t *ptep, int full) { - pte_t pte = *ptep; + pgste_t pgste; + pte_t pte; + + if (mm_has_pgste(mm)) + pgste = pgste_get_lock(ptep); - if (full) - pte_clear(mm, addr, ptep); - else - ptep_invalidate(mm, addr, ptep); + pte = *ptep; + if (!full) + __ptep_ipte(address, ptep); + pte_val(*ptep) = _PAGE_TYPE_EMPTY; + + if (mm_has_pgste(mm)) { + pgste = pgste_update_all(&pte, pgste); + pgste_set_unlock(ptep, pgste); + } return pte; } #define __HAVE_ARCH_PTEP_SET_WRPROTECT -#define ptep_set_wrprotect(__mm, __addr, __ptep) \ -({ \ - pte_t __pte = *(__ptep); \ - if (pte_write(__pte)) { \ - (__mm)->context.flush_mm = 1; \ - if (atomic_read(&(__mm)->context.attach_count) > 1 || \ - (__mm) != current->active_mm) \ - ptep_invalidate(__mm, __addr, __ptep); \ - set_pte_at(__mm, __addr, __ptep, pte_wrprotect(__pte)); \ - } \ -}) +static inline pte_t ptep_set_wrprotect(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + pgste_t pgste; + pte_t pte = *ptep; -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -#define ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ -({ \ - int __changed = !pte_same(*(__ptep), __entry); \ - if (__changed) { \ - ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \ - set_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \ - } \ - __changed; \ -}) + if (pte_write(pte)) { + mm->context.flush_mm = 1; + if (mm_has_pgste(mm)) + pgste = pgste_get_lock(ptep); -/* - * Test and clear dirty bit in storage key. - * We can't clear the changed bit atomically. This is a potential - * race against modification of the referenced bit. This function - * should therefore only be called if it is not mapped in any - * address space. - */ -#define __HAVE_ARCH_PAGE_TEST_DIRTY -static inline int page_test_dirty(struct page *page) -{ - return (page_get_storage_key(page_to_phys(page)) & _PAGE_CHANGED) != 0; -} + if (!mm_exclusive(mm)) + __ptep_ipte(address, ptep); + *ptep = pte_wrprotect(pte); -#define __HAVE_ARCH_PAGE_CLEAR_DIRTY -static inline void page_clear_dirty(struct page *page, int mapped) -{ - page_set_storage_key(page_to_phys(page), PAGE_DEFAULT_KEY, mapped); + if (mm_has_pgste(mm)) + pgste_set_unlock(ptep, pgste); + } + return pte; } -/* - * Test and clear referenced bit in storage key. - */ -#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG -static inline int page_test_and_clear_young(struct page *page) +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty) { - unsigned long physpage = page_to_phys(page); - int ccode; - - asm volatile( - " rrbe 0,%1\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (ccode) : "a" (physpage) : "cc" ); - return ccode & 2; + pgste_t pgste; + + if (pte_same(*ptep, entry)) + return 0; + if (mm_has_pgste(vma->vm_mm)) + pgste = pgste_get_lock(ptep); + + __ptep_ipte(address, ptep); + *ptep = entry; + + if (mm_has_pgste(vma->vm_mm)) + pgste_set_unlock(ptep, pgste); + return 1; } /* diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 2c79b6416271..a4b6229e5d4b 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -80,10 +80,12 @@ struct thread_struct { mm_segment_t mm_segment; unsigned long prot_addr; /* address of protection-excep. */ unsigned int trap_no; + unsigned long gmap_addr; /* address of last gmap fault. */ struct per_regs per_user; /* User specified PER registers */ struct per_event per_event; /* Cause of the last PER trap */ /* pfault_wait is used to block the process on a pfault event */ unsigned long pfault_wait; + struct list_head list; }; typedef struct thread_struct thread_struct; @@ -117,14 +119,12 @@ struct stack_frame { * Do necessary setup to start up a new thread. */ #define start_thread(regs, new_psw, new_stackp) do { \ - set_fs(USER_DS); \ regs->psw.mask = psw_user_bits; \ regs->psw.addr = new_psw | PSW_ADDR_AMODE; \ regs->gprs[15] = new_stackp; \ } while (0) #define start_thread31(regs, new_psw, new_stackp) do { \ - set_fs(USER_DS); \ regs->psw.mask = psw_user32_bits; \ regs->psw.addr = new_psw | PSW_ADDR_AMODE; \ regs->gprs[15] = new_stackp; \ diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 9ad628a8574a..62fd80c9e98c 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -539,7 +539,6 @@ struct user_regs_struct * These are defined as per linux/ptrace.h, which see. */ #define arch_has_single_step() (1) -extern void show_regs(struct pt_regs * regs); #define user_mode(regs) (((regs)->psw.mask & PSW_MASK_PSTATE) != 0) #define instruction_pointer(regs) ((regs)->psw.addr & PSW_ADDR_INSN) diff --git a/arch/s390/include/asm/qdio.h b/arch/s390/include/asm/qdio.h index 350e7ee5952d..15c97625df8d 100644 --- a/arch/s390/include/asm/qdio.h +++ b/arch/s390/include/asm/qdio.h @@ -139,110 +139,47 @@ struct slib { struct slibe slibe[QDIO_MAX_BUFFERS_PER_Q]; } __attribute__ ((packed, aligned(2048))); -/** - * struct sbal_flags - storage block address list flags - * @last: last entry - * @cont: contiguous storage - * @frag: fragmentation - */ -struct sbal_flags { - u8 : 1; - u8 last : 1; - u8 cont : 1; - u8 : 1; - u8 frag : 2; - u8 : 2; -} __attribute__ ((packed)); - -#define SBAL_FLAGS_FIRST_FRAG 0x04000000UL -#define SBAL_FLAGS_MIDDLE_FRAG 0x08000000UL -#define SBAL_FLAGS_LAST_FRAG 0x0c000000UL -#define SBAL_FLAGS_LAST_ENTRY 0x40000000UL -#define SBAL_FLAGS_CONTIGUOUS 0x20000000UL +#define SBAL_EFLAGS_LAST_ENTRY 0x40 +#define SBAL_EFLAGS_CONTIGUOUS 0x20 +#define SBAL_EFLAGS_FIRST_FRAG 0x04 +#define SBAL_EFLAGS_MIDDLE_FRAG 0x08 +#define SBAL_EFLAGS_LAST_FRAG 0x0c +#define SBAL_EFLAGS_MASK 0x6f -#define SBAL_FLAGS0_DATA_CONTINUATION 0x20UL +#define SBAL_SFLAGS0_PCI_REQ 0x40 +#define SBAL_SFLAGS0_DATA_CONTINUATION 0x20 /* Awesome OpenFCP extensions */ -#define SBAL_FLAGS0_TYPE_STATUS 0x00UL -#define SBAL_FLAGS0_TYPE_WRITE 0x08UL -#define SBAL_FLAGS0_TYPE_READ 0x10UL -#define SBAL_FLAGS0_TYPE_WRITE_READ 0x18UL -#define SBAL_FLAGS0_MORE_SBALS 0x04UL -#define SBAL_FLAGS0_COMMAND 0x02UL -#define SBAL_FLAGS0_LAST_SBAL 0x00UL -#define SBAL_FLAGS0_ONLY_SBAL SBAL_FLAGS0_COMMAND -#define SBAL_FLAGS0_MIDDLE_SBAL SBAL_FLAGS0_MORE_SBALS -#define SBAL_FLAGS0_FIRST_SBAL SBAL_FLAGS0_MORE_SBALS | SBAL_FLAGS0_COMMAND -#define SBAL_FLAGS0_PCI 0x40 - -/** - * struct sbal_sbalf_0 - sbal flags for sbale 0 - * @pci: PCI indicator - * @cont: data continuation - * @sbtype: storage-block type (FCP) - */ -struct sbal_sbalf_0 { - u8 : 1; - u8 pci : 1; - u8 cont : 1; - u8 sbtype : 2; - u8 : 3; -} __attribute__ ((packed)); - -/** - * struct sbal_sbalf_1 - sbal flags for sbale 1 - * @key: storage key - */ -struct sbal_sbalf_1 { - u8 : 4; - u8 key : 4; -} __attribute__ ((packed)); - -/** - * struct sbal_sbalf_14 - sbal flags for sbale 14 - * @erridx: error index - */ -struct sbal_sbalf_14 { - u8 : 4; - u8 erridx : 4; -} __attribute__ ((packed)); - -/** - * struct sbal_sbalf_15 - sbal flags for sbale 15 - * @reason: reason for error state - */ -struct sbal_sbalf_15 { - u8 reason; -} __attribute__ ((packed)); - -/** - * union sbal_sbalf - storage block address list flags - * @i0: sbalf0 - * @i1: sbalf1 - * @i14: sbalf14 - * @i15: sblaf15 - * @value: raw value - */ -union sbal_sbalf { - struct sbal_sbalf_0 i0; - struct sbal_sbalf_1 i1; - struct sbal_sbalf_14 i14; - struct sbal_sbalf_15 i15; - u8 value; -}; +#define SBAL_SFLAGS0_TYPE_STATUS 0x00 +#define SBAL_SFLAGS0_TYPE_WRITE 0x08 +#define SBAL_SFLAGS0_TYPE_READ 0x10 +#define SBAL_SFLAGS0_TYPE_WRITE_READ 0x18 +#define SBAL_SFLAGS0_MORE_SBALS 0x04 +#define SBAL_SFLAGS0_COMMAND 0x02 +#define SBAL_SFLAGS0_LAST_SBAL 0x00 +#define SBAL_SFLAGS0_ONLY_SBAL SBAL_SFLAGS0_COMMAND +#define SBAL_SFLAGS0_MIDDLE_SBAL SBAL_SFLAGS0_MORE_SBALS +#define SBAL_SFLAGS0_FIRST_SBAL (SBAL_SFLAGS0_MORE_SBALS | SBAL_SFLAGS0_COMMAND) /** * struct qdio_buffer_element - SBAL entry - * @flags: flags + * @eflags: SBAL entry flags + * @scount: SBAL count + * @sflags: whole SBAL flags * @length: length * @addr: address */ struct qdio_buffer_element { - u32 flags; + u8 eflags; + /* private: */ + u8 res1; + /* public: */ + u8 scount; + u8 sflags; u32 length; #ifdef CONFIG_32BIT /* private: */ - void *reserved; + void *res2; /* public: */ #endif void *addr; diff --git a/arch/s390/include/asm/s390_ext.h b/arch/s390/include/asm/s390_ext.h deleted file mode 100644 index 080876d5f196..000000000000 --- a/arch/s390/include/asm/s390_ext.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright IBM Corp. 1999,2010 - * Author(s): Holger Smolinski <Holger.Smolinski@de.ibm.com>, - * Martin Schwidefsky <schwidefsky@de.ibm.com>, - */ - -#ifndef _S390_EXTINT_H -#define _S390_EXTINT_H - -#include <linux/types.h> - -typedef void (*ext_int_handler_t)(unsigned int, unsigned int, unsigned long); - -int register_external_interrupt(__u16 code, ext_int_handler_t handler); -int unregister_external_interrupt(__u16 code, ext_int_handler_t handler); - -#endif /* _S390_EXTINT_H */ diff --git a/arch/s390/include/asm/suspend.h b/arch/s390/include/asm/suspend.h deleted file mode 100644 index dc75c616eafe..000000000000 --- a/arch/s390/include/asm/suspend.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef __ASM_S390_SUSPEND_H -#define __ASM_S390_SUSPEND_H - -static inline int arch_prepare_suspend(void) -{ - return 0; -} - -#endif - diff --git a/arch/s390/include/asm/system.h b/arch/s390/include/asm/system.h index d382629a0172..6582f69f2389 100644 --- a/arch/s390/include/asm/system.h +++ b/arch/s390/include/asm/system.h @@ -113,6 +113,7 @@ extern void pfault_fini(void); extern void cmma_init(void); extern int memcpy_real(void *, void *, size_t); +extern void copy_to_absolute_zero(void *dest, void *src, size_t count); #define finish_arch_switch(prev) do { \ set_fs(current->thread.mm_segment); \ diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index ad1382f7932e..1a5dbb6f1495 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -94,6 +94,7 @@ static inline struct thread_info *current_thread_info(void) #define TIF_SYSCALL_AUDIT 9 /* syscall auditing active */ #define TIF_SECCOMP 10 /* secure computing */ #define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ +#define TIF_SIE 12 /* guest execution active */ #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_31BIT 17 /* 32bit process */ @@ -113,6 +114,7 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1<<TIF_SECCOMP) #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) +#define _TIF_SIE (1<<TIF_SIE) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_31BIT (1<<TIF_31BIT) #define _TIF_SINGLE_STEP (1<<TIF_FREEZE) diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 9074a54c4d10..c687a2c83462 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -26,68 +26,73 @@ #include <linux/swap.h> #include <asm/processor.h> #include <asm/pgalloc.h> -#include <asm/smp.h> #include <asm/tlbflush.h> -#ifndef CONFIG_SMP -#define TLB_NR_PTRS 1 -#else -#define TLB_NR_PTRS 508 -#endif - struct mmu_gather { struct mm_struct *mm; +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + struct mmu_table_batch *batch; +#endif unsigned int fullmm; - unsigned int nr_ptes; - unsigned int nr_pxds; - void *array[TLB_NR_PTRS]; + unsigned int need_flush; }; -DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE +struct mmu_table_batch { + struct rcu_head rcu; + unsigned int nr; + void *tables[0]; +}; -static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, - unsigned int full_mm_flush) -{ - struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); +#define MAX_TABLE_BATCH \ + ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) +extern void tlb_table_flush(struct mmu_gather *tlb); +extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#endif + +static inline void tlb_gather_mmu(struct mmu_gather *tlb, + struct mm_struct *mm, + unsigned int full_mm_flush) +{ tlb->mm = mm; tlb->fullmm = full_mm_flush; - tlb->nr_ptes = 0; - tlb->nr_pxds = TLB_NR_PTRS; + tlb->need_flush = 0; +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb->batch = NULL; +#endif if (tlb->fullmm) __tlb_flush_mm(mm); - return tlb; } -static inline void tlb_flush_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end) +static inline void tlb_flush_mmu(struct mmu_gather *tlb) { - if (!tlb->fullmm && (tlb->nr_ptes > 0 || tlb->nr_pxds < TLB_NR_PTRS)) - __tlb_flush_mm(tlb->mm); - while (tlb->nr_ptes > 0) - page_table_free_rcu(tlb->mm, tlb->array[--tlb->nr_ptes]); - while (tlb->nr_pxds < TLB_NR_PTRS) - crst_table_free_rcu(tlb->mm, tlb->array[tlb->nr_pxds++]); + if (!tlb->need_flush) + return; + tlb->need_flush = 0; + __tlb_flush_mm(tlb->mm); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb_table_flush(tlb); +#endif } static inline void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) { - tlb_flush_mmu(tlb, start, end); - - rcu_table_freelist_finish(); - - /* keep the page table cache within bounds */ - check_pgt_cache(); - - put_cpu_var(mmu_gathers); + tlb_flush_mmu(tlb); } /* * Release the page cache reference for a pte removed by - * tlb_ptep_clear_flush. In both flush modes the tlb fo a page cache page + * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page * has already been freed, so just do free_page_and_swap_cache. */ +static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + free_page_and_swap_cache(page); + return 1; /* avoid calling tlb_flush_mmu */ +} + static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) { free_page_and_swap_cache(page); @@ -100,12 +105,11 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long address) { - if (!tlb->fullmm) { - tlb->array[tlb->nr_ptes++] = pte; - if (tlb->nr_ptes >= tlb->nr_pxds) - tlb_flush_mmu(tlb, 0, 0); - } else - page_table_free(tlb->mm, (unsigned long *) pte); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + if (!tlb->fullmm) + return page_table_free_rcu(tlb, (unsigned long *) pte); +#endif + page_table_free(tlb->mm, (unsigned long *) pte); } /* @@ -121,12 +125,11 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, #ifdef __s390x__ if (tlb->mm->context.asce_limit <= (1UL << 31)) return; - if (!tlb->fullmm) { - tlb->array[--tlb->nr_pxds] = pmd; - if (tlb->nr_ptes >= tlb->nr_pxds) - tlb_flush_mmu(tlb, 0, 0); - } else - crst_table_free(tlb->mm, (unsigned long *) pmd); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + if (!tlb->fullmm) + return tlb_remove_table(tlb, pmd); +#endif + crst_table_free(tlb->mm, (unsigned long *) pmd); #endif } @@ -143,12 +146,11 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, #ifdef __s390x__ if (tlb->mm->context.asce_limit <= (1UL << 42)) return; - if (!tlb->fullmm) { - tlb->array[--tlb->nr_pxds] = pud; - if (tlb->nr_ptes >= tlb->nr_pxds) - tlb_flush_mmu(tlb, 0, 0); - } else - crst_table_free(tlb->mm, (unsigned long *) pud); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + if (!tlb->fullmm) + return tlb_remove_table(tlb, pud); +#endif + crst_table_free(tlb->mm, (unsigned long *) pud); #endif } diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 29d5d6d4becc..304445382382 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -50,7 +50,7 @@ static inline void __tlb_flush_full(struct mm_struct *mm) /* * If the process only ran on the local cpu, do a local flush. */ - local_cpumask = cpumask_of_cpu(smp_processor_id()); + cpumask_copy(&local_cpumask, cpumask_of(smp_processor_id())); if (cpumask_equal(mm_cpumask(mm), &local_cpumask)) __tlb_flush_local(); else @@ -80,16 +80,11 @@ static inline void __tlb_flush_mm(struct mm_struct * mm) * on all cpus instead of doing a local flush if the mm * only ran on the local cpu. */ - if (MACHINE_HAS_IDTE) { - if (mm->context.noexec) - __tlb_flush_idte((unsigned long) - get_shadow_table(mm->pgd) | - mm->context.asce_bits); + if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) __tlb_flush_idte((unsigned long) mm->pgd | mm->context.asce_bits); - return; - } - __tlb_flush_full(mm); + else + __tlb_flush_full(mm); } static inline void __tlb_flush_mm_cond(struct mm_struct * mm) diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h index c5338834ddbd..005d77d8ae2a 100644 --- a/arch/s390/include/asm/topology.h +++ b/arch/s390/include/asm/topology.h @@ -7,7 +7,7 @@ extern unsigned char cpu_core_id[NR_CPUS]; extern cpumask_t cpu_core_map[NR_CPUS]; -static inline const struct cpumask *cpu_coregroup_mask(unsigned int cpu) +static inline const struct cpumask *cpu_coregroup_mask(int cpu) { return &cpu_core_map[cpu]; } @@ -21,7 +21,7 @@ static inline const struct cpumask *cpu_coregroup_mask(unsigned int cpu) extern unsigned char cpu_book_id[NR_CPUS]; extern cpumask_t cpu_book_map[NR_CPUS]; -static inline const struct cpumask *cpu_book_mask(unsigned int cpu) +static inline const struct cpumask *cpu_book_mask(int cpu) { return &cpu_book_map[cpu]; } diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 2d9ea11f919a..2b23885e81e9 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -49,12 +49,13 @@ #define segment_eq(a,b) ((a).ar4 == (b).ar4) +#define __access_ok(addr, size) \ +({ \ + __chk_user_ptr(addr); \ + 1; \ +}) -static inline int __access_ok(const void __user *addr, unsigned long size) -{ - return 1; -} -#define access_ok(type,addr,size) __access_ok(addr,size) +#define access_ok(type, addr, size) __access_ok(addr, size) /* * The exception table consists of pairs of addresses: the first is the diff --git a/arch/s390/include/asm/unistd.h b/arch/s390/include/asm/unistd.h index e82152572377..404bdb9671b4 100644 --- a/arch/s390/include/asm/unistd.h +++ b/arch/s390/include/asm/unistd.h @@ -276,7 +276,8 @@ #define __NR_open_by_handle_at 336 #define __NR_clock_adjtime 337 #define __NR_syncfs 338 -#define NR_syscalls 339 +#define __NR_setns 339 +#define NR_syscalls 340 /* * There are some system calls that are not present on 64 bit, some @@ -385,6 +386,7 @@ /* Ignore system calls that are also reachable via sys_socket */ #define __IGNORE_recvmmsg +#define __IGNORE_sendmmsg #define __ARCH_WANT_IPC_PARSE_VERSION #define __ARCH_WANT_OLD_READDIR diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 64230bc392fa..df3732249baa 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -20,10 +20,10 @@ CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w -obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \ - processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \ - s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \ - vdso.o vtime.o sysinfo.o nmi.o sclp.o +obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o vtime.o \ + processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o \ + debug.o irq.o ipl.o dis.o diag.o mem_detect.o sclp.o vdso.o \ + sysinfo.o jump_label.o obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index fe03c140002a..532fd4322156 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -27,12 +27,9 @@ int main(void) BLANK(); DEFINE(__TASK_pid, offsetof(struct task_struct, pid)); BLANK(); - DEFINE(__THREAD_per_cause, - offsetof(struct task_struct, thread.per_event.cause)); - DEFINE(__THREAD_per_address, - offsetof(struct task_struct, thread.per_event.address)); - DEFINE(__THREAD_per_paid, - offsetof(struct task_struct, thread.per_event.paid)); + DEFINE(__THREAD_per_cause, offsetof(struct task_struct, thread.per_event.cause)); + DEFINE(__THREAD_per_address, offsetof(struct task_struct, thread.per_event.address)); + DEFINE(__THREAD_per_paid, offsetof(struct task_struct, thread.per_event.paid)); BLANK(); DEFINE(__TI_task, offsetof(struct thread_info, task)); DEFINE(__TI_domain, offsetof(struct thread_info, exec_domain)); @@ -124,13 +121,11 @@ int main(void) DEFINE(__LC_LAST_UPDATE_TIMER, offsetof(struct _lowcore, last_update_timer)); DEFINE(__LC_LAST_UPDATE_CLOCK, offsetof(struct _lowcore, last_update_clock)); DEFINE(__LC_CURRENT, offsetof(struct _lowcore, current_task)); + DEFINE(__LC_CURRENT_PID, offsetof(struct _lowcore, current_pid)); DEFINE(__LC_THREAD_INFO, offsetof(struct _lowcore, thread_info)); DEFINE(__LC_KERNEL_STACK, offsetof(struct _lowcore, kernel_stack)); DEFINE(__LC_ASYNC_STACK, offsetof(struct _lowcore, async_stack)); DEFINE(__LC_PANIC_STACK, offsetof(struct _lowcore, panic_stack)); - DEFINE(__LC_KERNEL_ASCE, offsetof(struct _lowcore, kernel_asce)); - DEFINE(__LC_USER_ASCE, offsetof(struct _lowcore, user_asce)); - DEFINE(__LC_USER_EXEC_ASCE, offsetof(struct _lowcore, user_exec_asce)); DEFINE(__LC_INT_CLOCK, offsetof(struct _lowcore, int_clock)); DEFINE(__LC_MCCK_CLOCK, offsetof(struct _lowcore, mcck_clock)); DEFINE(__LC_MACHINE_FLAGS, offsetof(struct _lowcore, machine_flags)); @@ -144,6 +139,7 @@ int main(void) DEFINE(__LC_FPREGS_SAVE_AREA, offsetof(struct _lowcore, floating_pt_save_area)); DEFINE(__LC_GPREGS_SAVE_AREA, offsetof(struct _lowcore, gpregs_save_area)); DEFINE(__LC_CREGS_SAVE_AREA, offsetof(struct _lowcore, cregs_save_area)); + DEFINE(__LC_SAVE_AREA_64, offsetof(struct _lowcore, save_area_64)); #ifdef CONFIG_32BIT DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, extended_save_area_addr)); #else /* CONFIG_32BIT */ @@ -153,7 +149,7 @@ int main(void) DEFINE(__LC_FP_CREG_SAVE_AREA, offsetof(struct _lowcore, fpt_creg_save_area)); DEFINE(__LC_LAST_BREAK, offsetof(struct _lowcore, breaking_event_addr)); DEFINE(__LC_VDSO_PER_CPU, offsetof(struct _lowcore, vdso_per_cpu_data)); - DEFINE(__LC_SIE_HOOK, offsetof(struct _lowcore, sie_hook)); + DEFINE(__LC_GMAP, offsetof(struct _lowcore, gmap)); DEFINE(__LC_CMF_HPP, offsetof(struct _lowcore, cmf_hpp)); #endif /* CONFIG_32BIT */ return 0; diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S index 15e46ca94335..255435663bf8 100644 --- a/arch/s390/kernel/base.S +++ b/arch/s390/kernel/base.S @@ -6,13 +6,13 @@ * Michael Holzheu <holzheu@de.ibm.com> */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> #ifdef CONFIG_64BIT - .globl s390_base_mcck_handler -s390_base_mcck_handler: +ENTRY(s390_base_mcck_handler) basr %r13,0 0: lg %r15,__LC_PANIC_STACK # load panic stack aghi %r15,-STACK_FRAME_OVERHEAD @@ -26,13 +26,13 @@ s390_base_mcck_handler: lpswe __LC_MCK_OLD_PSW .section .bss + .align 8 .globl s390_base_mcck_handler_fn s390_base_mcck_handler_fn: .quad 0 .previous - .globl s390_base_ext_handler -s390_base_ext_handler: +ENTRY(s390_base_ext_handler) stmg %r0,%r15,__LC_SAVE_AREA basr %r13,0 0: aghi %r15,-STACK_FRAME_OVERHEAD @@ -46,13 +46,13 @@ s390_base_ext_handler: lpswe __LC_EXT_OLD_PSW .section .bss + .align 8 .globl s390_base_ext_handler_fn s390_base_ext_handler_fn: .quad 0 .previous - .globl s390_base_pgm_handler -s390_base_pgm_handler: +ENTRY(s390_base_pgm_handler) stmg %r0,%r15,__LC_SAVE_AREA basr %r13,0 0: aghi %r15,-STACK_FRAME_OVERHEAD @@ -70,15 +70,51 @@ disabled_wait_psw: .quad 0x0002000180000000,0x0000000000000000 + s390_base_pgm_handler .section .bss + .align 8 .globl s390_base_pgm_handler_fn s390_base_pgm_handler_fn: .quad 0 .previous +# +# Calls diag 308 subcode 1 and continues execution +# +# The following conditions must be ensured before calling this function: +# * Prefix register = 0 +# * Lowcore protection is disabled +# +ENTRY(diag308_reset) + larl %r4,.Lctlregs # Save control registers + stctg %c0,%c15,0(%r4) + larl %r4,.Lrestart_psw # Setup restart PSW at absolute 0 + lghi %r3,0 + lg %r4,0(%r4) # Save PSW + sturg %r4,%r3 # Use sturg, because of large pages + lghi %r1,1 + diag %r1,%r1,0x308 +.Lrestart_part2: + lhi %r0,0 # Load r0 with zero + lhi %r1,2 # Use mode 2 = ESAME (dump) + sigp %r1,%r0,0x12 # Switch to ESAME mode + sam64 # Switch to 64 bit addressing mode + larl %r4,.Lctlregs # Restore control registers + lctlg %c0,%c15,0(%r4) + br %r14 +.align 16 +.Lrestart_psw: + .long 0x00080000,0x80000000 + .Lrestart_part2 + + .section .bss +.align 8 +.Lctlregs: + .rept 16 + .quad 0 + .endr + .previous + #else /* CONFIG_64BIT */ - .globl s390_base_mcck_handler -s390_base_mcck_handler: +ENTRY(s390_base_mcck_handler) basr %r13,0 0: l %r15,__LC_PANIC_STACK # load panic stack ahi %r15,-STACK_FRAME_OVERHEAD @@ -93,13 +129,13 @@ s390_base_mcck_handler: 2: .long s390_base_mcck_handler_fn .section .bss + .align 4 .globl s390_base_mcck_handler_fn s390_base_mcck_handler_fn: .long 0 .previous - .globl s390_base_ext_handler -s390_base_ext_handler: +ENTRY(s390_base_ext_handler) stm %r0,%r15,__LC_SAVE_AREA basr %r13,0 0: ahi %r15,-STACK_FRAME_OVERHEAD @@ -115,13 +151,13 @@ s390_base_ext_handler: 2: .long s390_base_ext_handler_fn .section .bss + .align 4 .globl s390_base_ext_handler_fn s390_base_ext_handler_fn: .long 0 .previous - .globl s390_base_pgm_handler -s390_base_pgm_handler: +ENTRY(s390_base_pgm_handler) stm %r0,%r15,__LC_SAVE_AREA basr %r13,0 0: ahi %r15,-STACK_FRAME_OVERHEAD @@ -142,6 +178,7 @@ disabled_wait_psw: .long 0x000a0000,0x00000000 + s390_base_pgm_handler .section .bss + .align 4 .globl s390_base_pgm_handler_fn s390_base_pgm_handler_fn: .long 0 diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index eee999853a7c..a9a285b8c4ad 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -380,20 +380,13 @@ asmlinkage long sys32_sigreturn(void) goto badframe; if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE32)) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - + set_current_blocked(&set); if (restore_sigregs32(regs, &frame->sregs)) goto badframe; if (restore_sigregs_gprs_high(regs, frame->gprs_high)) goto badframe; - return regs->gprs[2]; - badframe: force_sig(SIGSEGV, current); return 0; @@ -413,31 +406,22 @@ asmlinkage long sys32_rt_sigreturn(void) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - + set_current_blocked(&set); if (restore_sigregs32(regs, &frame->uc.uc_mcontext)) goto badframe; if (restore_sigregs_gprs_high(regs, frame->gprs_high)) goto badframe; - err = __get_user(ss_sp, &frame->uc.uc_stack.ss_sp); st.ss_sp = compat_ptr(ss_sp); err |= __get_user(st.ss_size, &frame->uc.uc_stack.ss_size); err |= __get_user(st.ss_flags, &frame->uc.uc_stack.ss_flags); if (err) goto badframe; - set_fs (KERNEL_DS); do_sigaltstack((stack_t __force __user *)&st, NULL, regs->gprs[15]); set_fs (old_fs); - return regs->gprs[2]; - badframe: force_sig(SIGSEGV, current); return 0; @@ -605,10 +589,10 @@ give_sigsegv: * OK, we're invoking a handler */ -int -handle_signal32(unsigned long sig, struct k_sigaction *ka, - siginfo_t *info, sigset_t *oldset, struct pt_regs * regs) +int handle_signal32(unsigned long sig, struct k_sigaction *ka, + siginfo_t *info, sigset_t *oldset, struct pt_regs *regs) { + sigset_t blocked; int ret; /* Set up the stack frame */ @@ -616,15 +600,12 @@ handle_signal32(unsigned long sig, struct k_sigaction *ka, ret = setup_rt_frame32(sig, ka, info, oldset, regs); else ret = setup_frame32(sig, ka, oldset, regs); - - if (ret == 0) { - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NODEFER)) - sigaddset(¤t->blocked,sig); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - } - return ret; + if (ret) + return ret; + sigorsets(&blocked, ¤t->blocked, &ka->sa.sa_mask); + if (!(ka->sa.sa_flags & SA_NODEFER)) + sigaddset(&blocked, sig); + set_current_blocked(&blocked); + return 0; } diff --git a/arch/s390/kernel/compat_wrapper.S b/arch/s390/kernel/compat_wrapper.S index 1dc96ea08fa8..08ab9aa6a0d5 100644 --- a/arch/s390/kernel/compat_wrapper.S +++ b/arch/s390/kernel/compat_wrapper.S @@ -7,86 +7,74 @@ * Thomas Spatzier (tspat@de.ibm.com) */ - .globl sys32_exit_wrapper -sys32_exit_wrapper: +#include <linux/linkage.h> + +ENTRY(sys32_exit_wrapper) lgfr %r2,%r2 # int jg sys_exit # branch to sys_exit - .globl sys32_read_wrapper -sys32_read_wrapper: +ENTRY(sys32_read_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t jg sys32_read # branch to sys_read - .globl sys32_write_wrapper -sys32_write_wrapper: +ENTRY(sys32_write_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # size_t jg sys32_write # branch to system call - .globl sys32_open_wrapper -sys32_open_wrapper: +ENTRY(sys32_open_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int lgfr %r4,%r4 # int jg sys_open # branch to system call - .globl sys32_close_wrapper -sys32_close_wrapper: +ENTRY(sys32_close_wrapper) llgfr %r2,%r2 # unsigned int jg sys_close # branch to system call - .globl sys32_creat_wrapper -sys32_creat_wrapper: +ENTRY(sys32_creat_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int jg sys_creat # branch to system call - .globl sys32_link_wrapper -sys32_link_wrapper: +ENTRY(sys32_link_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * jg sys_link # branch to system call - .globl sys32_unlink_wrapper -sys32_unlink_wrapper: +ENTRY(sys32_unlink_wrapper) llgtr %r2,%r2 # const char * jg sys_unlink # branch to system call - .globl sys32_chdir_wrapper -sys32_chdir_wrapper: +ENTRY(sys32_chdir_wrapper) llgtr %r2,%r2 # const char * jg sys_chdir # branch to system call - .globl sys32_time_wrapper -sys32_time_wrapper: +ENTRY(sys32_time_wrapper) llgtr %r2,%r2 # int * jg compat_sys_time # branch to system call - .globl sys32_mknod_wrapper -sys32_mknod_wrapper: +ENTRY(sys32_mknod_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int llgfr %r4,%r4 # dev jg sys_mknod # branch to system call - .globl sys32_chmod_wrapper -sys32_chmod_wrapper: +ENTRY(sys32_chmod_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # mode_t jg sys_chmod # branch to system call - .globl sys32_lchown16_wrapper -sys32_lchown16_wrapper: +ENTRY(sys32_lchown16_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # __kernel_old_uid_emu31_t llgfr %r4,%r4 # __kernel_old_uid_emu31_t jg sys32_lchown16 # branch to system call - .globl sys32_lseek_wrapper -sys32_lseek_wrapper: +ENTRY(sys32_lseek_wrapper) llgfr %r2,%r2 # unsigned int lgfr %r3,%r3 # off_t llgfr %r4,%r4 # unsigned int @@ -94,8 +82,7 @@ sys32_lseek_wrapper: #sys32_getpid_wrapper # void - .globl sys32_mount_wrapper -sys32_mount_wrapper: +ENTRY(sys32_mount_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgtr %r4,%r4 # char * @@ -103,102 +90,85 @@ sys32_mount_wrapper: llgtr %r6,%r6 # void * jg compat_sys_mount # branch to system call - .globl sys32_oldumount_wrapper -sys32_oldumount_wrapper: +ENTRY(sys32_oldumount_wrapper) llgtr %r2,%r2 # char * jg sys_oldumount # branch to system call - .globl sys32_setuid16_wrapper -sys32_setuid16_wrapper: +ENTRY(sys32_setuid16_wrapper) llgfr %r2,%r2 # __kernel_old_uid_emu31_t jg sys32_setuid16 # branch to system call #sys32_getuid16_wrapper # void - .globl sys32_ptrace_wrapper -sys32_ptrace_wrapper: +ENTRY(sys32_ptrace_wrapper) lgfr %r2,%r2 # long lgfr %r3,%r3 # long llgtr %r4,%r4 # long llgfr %r5,%r5 # long jg compat_sys_ptrace # branch to system call - .globl sys32_alarm_wrapper -sys32_alarm_wrapper: +ENTRY(sys32_alarm_wrapper) llgfr %r2,%r2 # unsigned int jg sys_alarm # branch to system call - .globl compat_sys_utime_wrapper -compat_sys_utime_wrapper: +ENTRY(compat_sys_utime_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct compat_utimbuf * jg compat_sys_utime # branch to system call - .globl sys32_access_wrapper -sys32_access_wrapper: +ENTRY(sys32_access_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int jg sys_access # branch to system call - .globl sys32_nice_wrapper -sys32_nice_wrapper: +ENTRY(sys32_nice_wrapper) lgfr %r2,%r2 # int jg sys_nice # branch to system call #sys32_sync_wrapper # void - .globl sys32_kill_wrapper -sys32_kill_wrapper: +ENTRY(sys32_kill_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int jg sys_kill # branch to system call - .globl sys32_rename_wrapper -sys32_rename_wrapper: +ENTRY(sys32_rename_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * jg sys_rename # branch to system call - .globl sys32_mkdir_wrapper -sys32_mkdir_wrapper: +ENTRY(sys32_mkdir_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int jg sys_mkdir # branch to system call - .globl sys32_rmdir_wrapper -sys32_rmdir_wrapper: +ENTRY(sys32_rmdir_wrapper) llgtr %r2,%r2 # const char * jg sys_rmdir # branch to system call - .globl sys32_dup_wrapper -sys32_dup_wrapper: +ENTRY(sys32_dup_wrapper) llgfr %r2,%r2 # unsigned int jg sys_dup # branch to system call - .globl sys32_pipe_wrapper -sys32_pipe_wrapper: +ENTRY(sys32_pipe_wrapper) llgtr %r2,%r2 # u32 * jg sys_pipe # branch to system call - .globl compat_sys_times_wrapper -compat_sys_times_wrapper: +ENTRY(compat_sys_times_wrapper) llgtr %r2,%r2 # struct compat_tms * jg compat_sys_times # branch to system call - .globl sys32_brk_wrapper -sys32_brk_wrapper: +ENTRY(sys32_brk_wrapper) llgtr %r2,%r2 # unsigned long jg sys_brk # branch to system call - .globl sys32_setgid16_wrapper -sys32_setgid16_wrapper: +ENTRY(sys32_setgid16_wrapper) llgfr %r2,%r2 # __kernel_old_gid_emu31_t jg sys32_setgid16 # branch to system call #sys32_getgid16_wrapper # void - .globl sys32_signal_wrapper -sys32_signal_wrapper: +ENTRY(sys32_signal_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # __sighandler_t jg sys_signal @@ -207,55 +177,46 @@ sys32_signal_wrapper: #sys32_getegid16_wrapper # void - .globl sys32_acct_wrapper -sys32_acct_wrapper: +ENTRY(sys32_acct_wrapper) llgtr %r2,%r2 # char * jg sys_acct # branch to system call - .globl sys32_umount_wrapper -sys32_umount_wrapper: +ENTRY(sys32_umount_wrapper) llgtr %r2,%r2 # char * lgfr %r3,%r3 # int jg sys_umount # branch to system call - .globl compat_sys_ioctl_wrapper -compat_sys_ioctl_wrapper: +ENTRY(compat_sys_ioctl_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int llgfr %r4,%r4 # unsigned int jg compat_sys_ioctl # branch to system call - .globl compat_sys_fcntl_wrapper -compat_sys_fcntl_wrapper: +ENTRY(compat_sys_fcntl_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int llgfr %r4,%r4 # unsigned long jg compat_sys_fcntl # branch to system call - .globl sys32_setpgid_wrapper -sys32_setpgid_wrapper: +ENTRY(sys32_setpgid_wrapper) lgfr %r2,%r2 # pid_t lgfr %r3,%r3 # pid_t jg sys_setpgid # branch to system call - .globl sys32_umask_wrapper -sys32_umask_wrapper: +ENTRY(sys32_umask_wrapper) lgfr %r2,%r2 # int jg sys_umask # branch to system call - .globl sys32_chroot_wrapper -sys32_chroot_wrapper: +ENTRY(sys32_chroot_wrapper) llgtr %r2,%r2 # char * jg sys_chroot # branch to system call - .globl sys32_ustat_wrapper -sys32_ustat_wrapper: +ENTRY(sys32_ustat_wrapper) llgfr %r2,%r2 # dev_t llgtr %r3,%r3 # struct ustat * jg compat_sys_ustat - .globl sys32_dup2_wrapper -sys32_dup2_wrapper: +ENTRY(sys32_dup2_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int jg sys_dup2 # branch to system call @@ -266,262 +227,220 @@ sys32_dup2_wrapper: #sys32_setsid_wrapper # void - .globl sys32_sigaction_wrapper -sys32_sigaction_wrapper: +ENTRY(sys32_sigaction_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const struct old_sigaction * llgtr %r4,%r4 # struct old_sigaction32 * jg sys32_sigaction # branch to system call - .globl sys32_setreuid16_wrapper -sys32_setreuid16_wrapper: +ENTRY(sys32_setreuid16_wrapper) llgfr %r2,%r2 # __kernel_old_uid_emu31_t llgfr %r3,%r3 # __kernel_old_uid_emu31_t jg sys32_setreuid16 # branch to system call - .globl sys32_setregid16_wrapper -sys32_setregid16_wrapper: +ENTRY(sys32_setregid16_wrapper) llgfr %r2,%r2 # __kernel_old_gid_emu31_t llgfr %r3,%r3 # __kernel_old_gid_emu31_t jg sys32_setregid16 # branch to system call - .globl sys_sigsuspend_wrapper -sys_sigsuspend_wrapper: +ENTRY(sys_sigsuspend_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgfr %r4,%r4 # old_sigset_t jg sys_sigsuspend - .globl compat_sys_sigpending_wrapper -compat_sys_sigpending_wrapper: +ENTRY(compat_sys_sigpending_wrapper) llgtr %r2,%r2 # compat_old_sigset_t * jg compat_sys_sigpending # branch to system call - .globl sys32_sethostname_wrapper -sys32_sethostname_wrapper: +ENTRY(sys32_sethostname_wrapper) llgtr %r2,%r2 # char * lgfr %r3,%r3 # int jg sys_sethostname # branch to system call - .globl compat_sys_setrlimit_wrapper -compat_sys_setrlimit_wrapper: +ENTRY(compat_sys_setrlimit_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # struct rlimit_emu31 * jg compat_sys_setrlimit # branch to system call - .globl compat_sys_old_getrlimit_wrapper -compat_sys_old_getrlimit_wrapper: +ENTRY(compat_sys_old_getrlimit_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # struct rlimit_emu31 * jg compat_sys_old_getrlimit # branch to system call - .globl compat_sys_getrlimit_wrapper -compat_sys_getrlimit_wrapper: +ENTRY(compat_sys_getrlimit_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # struct rlimit_emu31 * jg compat_sys_getrlimit # branch to system call - .globl sys32_mmap2_wrapper -sys32_mmap2_wrapper: +ENTRY(sys32_mmap2_wrapper) llgtr %r2,%r2 # struct mmap_arg_struct_emu31 * jg sys32_mmap2 # branch to system call - .globl compat_sys_getrusage_wrapper -compat_sys_getrusage_wrapper: +ENTRY(compat_sys_getrusage_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct rusage_emu31 * jg compat_sys_getrusage # branch to system call - .globl compat_sys_gettimeofday_wrapper -compat_sys_gettimeofday_wrapper: +ENTRY(compat_sys_gettimeofday_wrapper) llgtr %r2,%r2 # struct timeval_emu31 * llgtr %r3,%r3 # struct timezone * jg compat_sys_gettimeofday # branch to system call - .globl compat_sys_settimeofday_wrapper -compat_sys_settimeofday_wrapper: +ENTRY(compat_sys_settimeofday_wrapper) llgtr %r2,%r2 # struct timeval_emu31 * llgtr %r3,%r3 # struct timezone * jg compat_sys_settimeofday # branch to system call - .globl sys32_getgroups16_wrapper -sys32_getgroups16_wrapper: +ENTRY(sys32_getgroups16_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # __kernel_old_gid_emu31_t * jg sys32_getgroups16 # branch to system call - .globl sys32_setgroups16_wrapper -sys32_setgroups16_wrapper: +ENTRY(sys32_setgroups16_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # __kernel_old_gid_emu31_t * jg sys32_setgroups16 # branch to system call - .globl sys32_symlink_wrapper -sys32_symlink_wrapper: +ENTRY(sys32_symlink_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * jg sys_symlink # branch to system call - .globl sys32_readlink_wrapper -sys32_readlink_wrapper: +ENTRY(sys32_readlink_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # char * lgfr %r4,%r4 # int jg sys_readlink # branch to system call - .globl sys32_uselib_wrapper -sys32_uselib_wrapper: +ENTRY(sys32_uselib_wrapper) llgtr %r2,%r2 # const char * jg sys_uselib # branch to system call - .globl sys32_swapon_wrapper -sys32_swapon_wrapper: +ENTRY(sys32_swapon_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int jg sys_swapon # branch to system call - .globl sys32_reboot_wrapper -sys32_reboot_wrapper: +ENTRY(sys32_reboot_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgfr %r4,%r4 # unsigned int llgtr %r5,%r5 # void * jg sys_reboot # branch to system call - .globl old32_readdir_wrapper -old32_readdir_wrapper: +ENTRY(old32_readdir_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # void * llgfr %r4,%r4 # unsigned int jg compat_sys_old_readdir # branch to system call - .globl old32_mmap_wrapper -old32_mmap_wrapper: +ENTRY(old32_mmap_wrapper) llgtr %r2,%r2 # struct mmap_arg_struct_emu31 * jg old32_mmap # branch to system call - .globl sys32_munmap_wrapper -sys32_munmap_wrapper: +ENTRY(sys32_munmap_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t jg sys_munmap # branch to system call - .globl sys32_truncate_wrapper -sys32_truncate_wrapper: +ENTRY(sys32_truncate_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # long jg sys_truncate # branch to system call - .globl sys32_ftruncate_wrapper -sys32_ftruncate_wrapper: +ENTRY(sys32_ftruncate_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned long jg sys_ftruncate # branch to system call - .globl sys32_fchmod_wrapper -sys32_fchmod_wrapper: +ENTRY(sys32_fchmod_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # mode_t jg sys_fchmod # branch to system call - .globl sys32_fchown16_wrapper -sys32_fchown16_wrapper: +ENTRY(sys32_fchown16_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # compat_uid_t llgfr %r4,%r4 # compat_uid_t jg sys32_fchown16 # branch to system call - .globl sys32_getpriority_wrapper -sys32_getpriority_wrapper: +ENTRY(sys32_getpriority_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int jg sys_getpriority # branch to system call - .globl sys32_setpriority_wrapper -sys32_setpriority_wrapper: +ENTRY(sys32_setpriority_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int lgfr %r4,%r4 # int jg sys_setpriority # branch to system call - .globl compat_sys_statfs_wrapper -compat_sys_statfs_wrapper: +ENTRY(compat_sys_statfs_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct compat_statfs * jg compat_sys_statfs # branch to system call - .globl compat_sys_fstatfs_wrapper -compat_sys_fstatfs_wrapper: +ENTRY(compat_sys_fstatfs_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # struct compat_statfs * jg compat_sys_fstatfs # branch to system call - .globl compat_sys_socketcall_wrapper -compat_sys_socketcall_wrapper: +ENTRY(compat_sys_socketcall_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # u32 * jg compat_sys_socketcall # branch to system call - .globl sys32_syslog_wrapper -sys32_syslog_wrapper: +ENTRY(sys32_syslog_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # char * lgfr %r4,%r4 # int jg sys_syslog # branch to system call - .globl compat_sys_setitimer_wrapper -compat_sys_setitimer_wrapper: +ENTRY(compat_sys_setitimer_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct itimerval_emu31 * llgtr %r4,%r4 # struct itimerval_emu31 * jg compat_sys_setitimer # branch to system call - .globl compat_sys_getitimer_wrapper -compat_sys_getitimer_wrapper: +ENTRY(compat_sys_getitimer_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct itimerval_emu31 * jg compat_sys_getitimer # branch to system call - .globl compat_sys_newstat_wrapper -compat_sys_newstat_wrapper: +ENTRY(compat_sys_newstat_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct stat_emu31 * jg compat_sys_newstat # branch to system call - .globl compat_sys_newlstat_wrapper -compat_sys_newlstat_wrapper: +ENTRY(compat_sys_newlstat_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct stat_emu31 * jg compat_sys_newlstat # branch to system call - .globl compat_sys_newfstat_wrapper -compat_sys_newfstat_wrapper: +ENTRY(compat_sys_newfstat_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # struct stat_emu31 * jg compat_sys_newfstat # branch to system call #sys32_vhangup_wrapper # void - .globl compat_sys_wait4_wrapper -compat_sys_wait4_wrapper: +ENTRY(compat_sys_wait4_wrapper) lgfr %r2,%r2 # pid_t llgtr %r3,%r3 # unsigned int * lgfr %r4,%r4 # int llgtr %r5,%r5 # struct rusage * jg compat_sys_wait4 # branch to system call - .globl sys32_swapoff_wrapper -sys32_swapoff_wrapper: +ENTRY(sys32_swapoff_wrapper) llgtr %r2,%r2 # const char * jg sys_swapoff # branch to system call - .globl compat_sys_sysinfo_wrapper -compat_sys_sysinfo_wrapper: +ENTRY(compat_sys_sysinfo_wrapper) llgtr %r2,%r2 # struct sysinfo_emu31 * jg compat_sys_sysinfo # branch to system call - .globl sys32_ipc_wrapper -sys32_ipc_wrapper: +ENTRY(sys32_ipc_wrapper) llgfr %r2,%r2 # uint lgfr %r3,%r3 # int lgfr %r4,%r4 # int @@ -529,8 +448,7 @@ sys32_ipc_wrapper: llgfr %r6,%r6 # u32 jg sys32_ipc # branch to system call - .globl sys32_fsync_wrapper -sys32_fsync_wrapper: +ENTRY(sys32_fsync_wrapper) llgfr %r2,%r2 # unsigned int jg sys_fsync # branch to system call @@ -538,97 +456,81 @@ sys32_fsync_wrapper: #sys32_clone_wrapper # done in clone_glue - .globl sys32_setdomainname_wrapper -sys32_setdomainname_wrapper: +ENTRY(sys32_setdomainname_wrapper) llgtr %r2,%r2 # char * lgfr %r3,%r3 # int jg sys_setdomainname # branch to system call - .globl sys32_newuname_wrapper -sys32_newuname_wrapper: +ENTRY(sys32_newuname_wrapper) llgtr %r2,%r2 # struct new_utsname * jg sys_newuname # branch to system call - .globl compat_sys_adjtimex_wrapper -compat_sys_adjtimex_wrapper: +ENTRY(compat_sys_adjtimex_wrapper) llgtr %r2,%r2 # struct compat_timex * jg compat_sys_adjtimex # branch to system call - .globl sys32_mprotect_wrapper -sys32_mprotect_wrapper: +ENTRY(sys32_mprotect_wrapper) llgtr %r2,%r2 # unsigned long (actually pointer llgfr %r3,%r3 # size_t llgfr %r4,%r4 # unsigned long jg sys_mprotect # branch to system call - .globl compat_sys_sigprocmask_wrapper -compat_sys_sigprocmask_wrapper: +ENTRY(compat_sys_sigprocmask_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_old_sigset_t * llgtr %r4,%r4 # compat_old_sigset_t * jg compat_sys_sigprocmask # branch to system call - .globl sys_init_module_wrapper -sys_init_module_wrapper: +ENTRY(sys_init_module_wrapper) llgtr %r2,%r2 # void * llgfr %r3,%r3 # unsigned long llgtr %r4,%r4 # char * jg sys_init_module # branch to system call - .globl sys_delete_module_wrapper -sys_delete_module_wrapper: +ENTRY(sys_delete_module_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # unsigned int jg sys_delete_module # branch to system call - .globl sys32_quotactl_wrapper -sys32_quotactl_wrapper: +ENTRY(sys32_quotactl_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # qid_t llgtr %r5,%r5 # caddr_t jg sys_quotactl # branch to system call - .globl sys32_getpgid_wrapper -sys32_getpgid_wrapper: +ENTRY(sys32_getpgid_wrapper) lgfr %r2,%r2 # pid_t jg sys_getpgid # branch to system call - .globl sys32_fchdir_wrapper -sys32_fchdir_wrapper: +ENTRY(sys32_fchdir_wrapper) llgfr %r2,%r2 # unsigned int jg sys_fchdir # branch to system call - .globl sys32_bdflush_wrapper -sys32_bdflush_wrapper: +ENTRY(sys32_bdflush_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # long jg sys_bdflush # branch to system call - .globl sys32_sysfs_wrapper -sys32_sysfs_wrapper: +ENTRY(sys32_sysfs_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long jg sys_sysfs # branch to system call - .globl sys32_personality_wrapper -sys32_personality_wrapper: +ENTRY(sys32_personality_wrapper) llgfr %r2,%r2 # unsigned int jg sys_s390_personality # branch to system call - .globl sys32_setfsuid16_wrapper -sys32_setfsuid16_wrapper: +ENTRY(sys32_setfsuid16_wrapper) llgfr %r2,%r2 # __kernel_old_uid_emu31_t jg sys32_setfsuid16 # branch to system call - .globl sys32_setfsgid16_wrapper -sys32_setfsgid16_wrapper: +ENTRY(sys32_setfsgid16_wrapper) llgfr %r2,%r2 # __kernel_old_gid_emu31_t jg sys32_setfsgid16 # branch to system call - .globl sys32_llseek_wrapper -sys32_llseek_wrapper: +ENTRY(sys32_llseek_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long @@ -636,15 +538,13 @@ sys32_llseek_wrapper: llgfr %r6,%r6 # unsigned int jg sys_llseek # branch to system call - .globl sys32_getdents_wrapper -sys32_getdents_wrapper: +ENTRY(sys32_getdents_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # void * llgfr %r4,%r4 # unsigned int jg compat_sys_getdents # branch to system call - .globl compat_sys_select_wrapper -compat_sys_select_wrapper: +ENTRY(compat_sys_select_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_fd_set * llgtr %r4,%r4 # compat_fd_set * @@ -652,112 +552,94 @@ compat_sys_select_wrapper: llgtr %r6,%r6 # struct compat_timeval * jg compat_sys_select # branch to system call - .globl sys32_flock_wrapper -sys32_flock_wrapper: +ENTRY(sys32_flock_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int jg sys_flock # branch to system call - .globl sys32_msync_wrapper -sys32_msync_wrapper: +ENTRY(sys32_msync_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t lgfr %r4,%r4 # int jg sys_msync # branch to system call - .globl compat_sys_readv_wrapper -compat_sys_readv_wrapper: +ENTRY(compat_sys_readv_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const struct compat_iovec * llgfr %r4,%r4 # unsigned long jg compat_sys_readv # branch to system call - .globl compat_sys_writev_wrapper -compat_sys_writev_wrapper: +ENTRY(compat_sys_writev_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const struct compat_iovec * llgfr %r4,%r4 # unsigned long jg compat_sys_writev # branch to system call - .globl sys32_getsid_wrapper -sys32_getsid_wrapper: +ENTRY(sys32_getsid_wrapper) lgfr %r2,%r2 # pid_t jg sys_getsid # branch to system call - .globl sys32_fdatasync_wrapper -sys32_fdatasync_wrapper: +ENTRY(sys32_fdatasync_wrapper) llgfr %r2,%r2 # unsigned int jg sys_fdatasync # branch to system call - .globl sys32_mlock_wrapper -sys32_mlock_wrapper: +ENTRY(sys32_mlock_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t jg sys_mlock # branch to system call - .globl sys32_munlock_wrapper -sys32_munlock_wrapper: +ENTRY(sys32_munlock_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t jg sys_munlock # branch to system call - .globl sys32_mlockall_wrapper -sys32_mlockall_wrapper: +ENTRY(sys32_mlockall_wrapper) lgfr %r2,%r2 # int jg sys_mlockall # branch to system call #sys32_munlockall_wrapper # void - .globl sys32_sched_setparam_wrapper -sys32_sched_setparam_wrapper: +ENTRY(sys32_sched_setparam_wrapper) lgfr %r2,%r2 # pid_t llgtr %r3,%r3 # struct sched_param * jg sys_sched_setparam # branch to system call - .globl sys32_sched_getparam_wrapper -sys32_sched_getparam_wrapper: +ENTRY(sys32_sched_getparam_wrapper) lgfr %r2,%r2 # pid_t llgtr %r3,%r3 # struct sched_param * jg sys_sched_getparam # branch to system call - .globl sys32_sched_setscheduler_wrapper -sys32_sched_setscheduler_wrapper: +ENTRY(sys32_sched_setscheduler_wrapper) lgfr %r2,%r2 # pid_t lgfr %r3,%r3 # int llgtr %r4,%r4 # struct sched_param * jg sys_sched_setscheduler # branch to system call - .globl sys32_sched_getscheduler_wrapper -sys32_sched_getscheduler_wrapper: +ENTRY(sys32_sched_getscheduler_wrapper) lgfr %r2,%r2 # pid_t jg sys_sched_getscheduler # branch to system call #sys32_sched_yield_wrapper # void - .globl sys32_sched_get_priority_max_wrapper -sys32_sched_get_priority_max_wrapper: +ENTRY(sys32_sched_get_priority_max_wrapper) lgfr %r2,%r2 # int jg sys_sched_get_priority_max # branch to system call - .globl sys32_sched_get_priority_min_wrapper -sys32_sched_get_priority_min_wrapper: +ENTRY(sys32_sched_get_priority_min_wrapper) lgfr %r2,%r2 # int jg sys_sched_get_priority_min # branch to system call - .globl sys32_sched_rr_get_interval_wrapper -sys32_sched_rr_get_interval_wrapper: +ENTRY(sys32_sched_rr_get_interval_wrapper) lgfr %r2,%r2 # pid_t llgtr %r3,%r3 # struct compat_timespec * jg sys32_sched_rr_get_interval # branch to system call - .globl compat_sys_nanosleep_wrapper -compat_sys_nanosleep_wrapper: +ENTRY(compat_sys_nanosleep_wrapper) llgtr %r2,%r2 # struct compat_timespec * llgtr %r3,%r3 # struct compat_timespec * jg compat_sys_nanosleep # branch to system call - .globl sys32_mremap_wrapper -sys32_mremap_wrapper: +ENTRY(sys32_mremap_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long @@ -765,50 +647,43 @@ sys32_mremap_wrapper: llgfr %r6,%r6 # unsigned long jg sys_mremap # branch to system call - .globl sys32_setresuid16_wrapper -sys32_setresuid16_wrapper: +ENTRY(sys32_setresuid16_wrapper) llgfr %r2,%r2 # __kernel_old_uid_emu31_t llgfr %r3,%r3 # __kernel_old_uid_emu31_t llgfr %r4,%r4 # __kernel_old_uid_emu31_t jg sys32_setresuid16 # branch to system call - .globl sys32_getresuid16_wrapper -sys32_getresuid16_wrapper: +ENTRY(sys32_getresuid16_wrapper) llgtr %r2,%r2 # __kernel_old_uid_emu31_t * llgtr %r3,%r3 # __kernel_old_uid_emu31_t * llgtr %r4,%r4 # __kernel_old_uid_emu31_t * jg sys32_getresuid16 # branch to system call - .globl sys32_poll_wrapper -sys32_poll_wrapper: +ENTRY(sys32_poll_wrapper) llgtr %r2,%r2 # struct pollfd * llgfr %r3,%r3 # unsigned int lgfr %r4,%r4 # long jg sys_poll # branch to system call - .globl compat_sys_nfsservctl_wrapper -compat_sys_nfsservctl_wrapper: +ENTRY(compat_sys_nfsservctl_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct compat_nfsctl_arg* llgtr %r4,%r4 # union compat_nfsctl_res* jg compat_sys_nfsservctl # branch to system call - .globl sys32_setresgid16_wrapper -sys32_setresgid16_wrapper: +ENTRY(sys32_setresgid16_wrapper) llgfr %r2,%r2 # __kernel_old_gid_emu31_t llgfr %r3,%r3 # __kernel_old_gid_emu31_t llgfr %r4,%r4 # __kernel_old_gid_emu31_t jg sys32_setresgid16 # branch to system call - .globl sys32_getresgid16_wrapper -sys32_getresgid16_wrapper: +ENTRY(sys32_getresgid16_wrapper) llgtr %r2,%r2 # __kernel_old_gid_emu31_t * llgtr %r3,%r3 # __kernel_old_gid_emu31_t * llgtr %r4,%r4 # __kernel_old_gid_emu31_t * jg sys32_getresgid16 # branch to system call - .globl sys32_prctl_wrapper -sys32_prctl_wrapper: +ENTRY(sys32_prctl_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long @@ -818,51 +693,44 @@ sys32_prctl_wrapper: #sys32_rt_sigreturn_wrapper # done in rt_sigreturn_glue - .globl sys32_rt_sigaction_wrapper -sys32_rt_sigaction_wrapper: +ENTRY(sys32_rt_sigaction_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const struct sigaction_emu31 * llgtr %r4,%r4 # const struct sigaction_emu31 * llgfr %r5,%r5 # size_t jg sys32_rt_sigaction # branch to system call - .globl sys32_rt_sigprocmask_wrapper -sys32_rt_sigprocmask_wrapper: +ENTRY(sys32_rt_sigprocmask_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # old_sigset_emu31 * llgtr %r4,%r4 # old_sigset_emu31 * llgfr %r5,%r5 # size_t jg sys32_rt_sigprocmask # branch to system call - .globl sys32_rt_sigpending_wrapper -sys32_rt_sigpending_wrapper: +ENTRY(sys32_rt_sigpending_wrapper) llgtr %r2,%r2 # sigset_emu31 * llgfr %r3,%r3 # size_t jg sys32_rt_sigpending # branch to system call - .globl compat_sys_rt_sigtimedwait_wrapper -compat_sys_rt_sigtimedwait_wrapper: +ENTRY(compat_sys_rt_sigtimedwait_wrapper) llgtr %r2,%r2 # const sigset_emu31_t * llgtr %r3,%r3 # siginfo_emu31_t * llgtr %r4,%r4 # const struct compat_timespec * llgfr %r5,%r5 # size_t jg compat_sys_rt_sigtimedwait # branch to system call - .globl sys32_rt_sigqueueinfo_wrapper -sys32_rt_sigqueueinfo_wrapper: +ENTRY(sys32_rt_sigqueueinfo_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgtr %r4,%r4 # siginfo_emu31_t * jg sys32_rt_sigqueueinfo # branch to system call - .globl compat_sys_rt_sigsuspend_wrapper -compat_sys_rt_sigsuspend_wrapper: +ENTRY(compat_sys_rt_sigsuspend_wrapper) llgtr %r2,%r2 # compat_sigset_t * llgfr %r3,%r3 # compat_size_t jg compat_sys_rt_sigsuspend - .globl sys32_pread64_wrapper -sys32_pread64_wrapper: +ENTRY(sys32_pread64_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t @@ -870,8 +738,7 @@ sys32_pread64_wrapper: llgfr %r6,%r6 # u32 jg sys32_pread64 # branch to system call - .globl sys32_pwrite64_wrapper -sys32_pwrite64_wrapper: +ENTRY(sys32_pwrite64_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # size_t @@ -879,39 +746,33 @@ sys32_pwrite64_wrapper: llgfr %r6,%r6 # u32 jg sys32_pwrite64 # branch to system call - .globl sys32_chown16_wrapper -sys32_chown16_wrapper: +ENTRY(sys32_chown16_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # __kernel_old_uid_emu31_t llgfr %r4,%r4 # __kernel_old_gid_emu31_t jg sys32_chown16 # branch to system call - .globl sys32_getcwd_wrapper -sys32_getcwd_wrapper: +ENTRY(sys32_getcwd_wrapper) llgtr %r2,%r2 # char * llgfr %r3,%r3 # unsigned long jg sys_getcwd # branch to system call - .globl sys32_capget_wrapper -sys32_capget_wrapper: +ENTRY(sys32_capget_wrapper) llgtr %r2,%r2 # cap_user_header_t llgtr %r3,%r3 # cap_user_data_t jg sys_capget # branch to system call - .globl sys32_capset_wrapper -sys32_capset_wrapper: +ENTRY(sys32_capset_wrapper) llgtr %r2,%r2 # cap_user_header_t llgtr %r3,%r3 # const cap_user_data_t jg sys_capset # branch to system call - .globl sys32_sigaltstack_wrapper -sys32_sigaltstack_wrapper: +ENTRY(sys32_sigaltstack_wrapper) llgtr %r2,%r2 # const stack_emu31_t * llgtr %r3,%r3 # stack_emu31_t * jg sys32_sigaltstack - .globl sys32_sendfile_wrapper -sys32_sendfile_wrapper: +ENTRY(sys32_sendfile_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgtr %r4,%r4 # __kernel_off_emu31_t * @@ -920,22 +781,19 @@ sys32_sendfile_wrapper: #sys32_vfork_wrapper # done in vfork_glue - .globl sys32_truncate64_wrapper -sys32_truncate64_wrapper: +ENTRY(sys32_truncate64_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long jg sys32_truncate64 # branch to system call - .globl sys32_ftruncate64_wrapper -sys32_ftruncate64_wrapper: +ENTRY(sys32_ftruncate64_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long jg sys32_ftruncate64 # branch to system call - .globl sys32_lchown_wrapper -sys32_lchown_wrapper: +ENTRY(sys32_lchown_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # uid_t llgfr %r4,%r4 # gid_t @@ -946,156 +804,131 @@ sys32_lchown_wrapper: #sys32_geteuid_wrapper # void #sys32_getegid_wrapper # void - .globl sys32_setreuid_wrapper -sys32_setreuid_wrapper: +ENTRY(sys32_setreuid_wrapper) llgfr %r2,%r2 # uid_t llgfr %r3,%r3 # uid_t jg sys_setreuid # branch to system call - .globl sys32_setregid_wrapper -sys32_setregid_wrapper: +ENTRY(sys32_setregid_wrapper) llgfr %r2,%r2 # gid_t llgfr %r3,%r3 # gid_t jg sys_setregid # branch to system call - .globl sys32_getgroups_wrapper -sys32_getgroups_wrapper: +ENTRY(sys32_getgroups_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # gid_t * jg sys_getgroups # branch to system call - .globl sys32_setgroups_wrapper -sys32_setgroups_wrapper: +ENTRY(sys32_setgroups_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # gid_t * jg sys_setgroups # branch to system call - .globl sys32_fchown_wrapper -sys32_fchown_wrapper: +ENTRY(sys32_fchown_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # uid_t llgfr %r4,%r4 # gid_t jg sys_fchown # branch to system call - .globl sys32_setresuid_wrapper -sys32_setresuid_wrapper: +ENTRY(sys32_setresuid_wrapper) llgfr %r2,%r2 # uid_t llgfr %r3,%r3 # uid_t llgfr %r4,%r4 # uid_t jg sys_setresuid # branch to system call - .globl sys32_getresuid_wrapper -sys32_getresuid_wrapper: +ENTRY(sys32_getresuid_wrapper) llgtr %r2,%r2 # uid_t * llgtr %r3,%r3 # uid_t * llgtr %r4,%r4 # uid_t * jg sys_getresuid # branch to system call - .globl sys32_setresgid_wrapper -sys32_setresgid_wrapper: +ENTRY(sys32_setresgid_wrapper) llgfr %r2,%r2 # gid_t llgfr %r3,%r3 # gid_t llgfr %r4,%r4 # gid_t jg sys_setresgid # branch to system call - .globl sys32_getresgid_wrapper -sys32_getresgid_wrapper: +ENTRY(sys32_getresgid_wrapper) llgtr %r2,%r2 # gid_t * llgtr %r3,%r3 # gid_t * llgtr %r4,%r4 # gid_t * jg sys_getresgid # branch to system call - .globl sys32_chown_wrapper -sys32_chown_wrapper: +ENTRY(sys32_chown_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # uid_t llgfr %r4,%r4 # gid_t jg sys_chown # branch to system call - .globl sys32_setuid_wrapper -sys32_setuid_wrapper: +ENTRY(sys32_setuid_wrapper) llgfr %r2,%r2 # uid_t jg sys_setuid # branch to system call - .globl sys32_setgid_wrapper -sys32_setgid_wrapper: +ENTRY(sys32_setgid_wrapper) llgfr %r2,%r2 # gid_t jg sys_setgid # branch to system call - .globl sys32_setfsuid_wrapper -sys32_setfsuid_wrapper: +ENTRY(sys32_setfsuid_wrapper) llgfr %r2,%r2 # uid_t jg sys_setfsuid # branch to system call - .globl sys32_setfsgid_wrapper -sys32_setfsgid_wrapper: +ENTRY(sys32_setfsgid_wrapper) llgfr %r2,%r2 # gid_t jg sys_setfsgid # branch to system call - .globl sys32_pivot_root_wrapper -sys32_pivot_root_wrapper: +ENTRY(sys32_pivot_root_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * jg sys_pivot_root # branch to system call - .globl sys32_mincore_wrapper -sys32_mincore_wrapper: +ENTRY(sys32_mincore_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t llgtr %r4,%r4 # unsigned char * jg sys_mincore # branch to system call - .globl sys32_madvise_wrapper -sys32_madvise_wrapper: +ENTRY(sys32_madvise_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # size_t lgfr %r4,%r4 # int jg sys_madvise # branch to system call - .globl sys32_getdents64_wrapper -sys32_getdents64_wrapper: +ENTRY(sys32_getdents64_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # void * llgfr %r4,%r4 # unsigned int jg sys_getdents64 # branch to system call - .globl compat_sys_fcntl64_wrapper -compat_sys_fcntl64_wrapper: +ENTRY(compat_sys_fcntl64_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int llgfr %r4,%r4 # unsigned long jg compat_sys_fcntl64 # branch to system call - .globl sys32_stat64_wrapper -sys32_stat64_wrapper: +ENTRY(sys32_stat64_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct stat64 * jg sys32_stat64 # branch to system call - .globl sys32_lstat64_wrapper -sys32_lstat64_wrapper: +ENTRY(sys32_lstat64_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct stat64 * jg sys32_lstat64 # branch to system call - .globl sys32_stime_wrapper -sys32_stime_wrapper: +ENTRY(sys32_stime_wrapper) llgtr %r2,%r2 # long * jg compat_sys_stime # branch to system call - .globl sys32_sysctl_wrapper -sys32_sysctl_wrapper: +ENTRY(sys32_sysctl_wrapper) llgtr %r2,%r2 # struct compat_sysctl_args * jg compat_sys_sysctl - .globl sys32_fstat64_wrapper -sys32_fstat64_wrapper: +ENTRY(sys32_fstat64_wrapper) llgfr %r2,%r2 # unsigned long llgtr %r3,%r3 # struct stat64 * jg sys32_fstat64 # branch to system call - .globl compat_sys_futex_wrapper -compat_sys_futex_wrapper: +ENTRY(compat_sys_futex_wrapper) llgtr %r2,%r2 # u32 * lgfr %r3,%r3 # int lgfr %r4,%r4 # int @@ -1105,8 +938,7 @@ compat_sys_futex_wrapper: stg %r0,160(%r15) jg compat_sys_futex # branch to system call - .globl sys32_setxattr_wrapper -sys32_setxattr_wrapper: +ENTRY(sys32_setxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * @@ -1114,8 +946,7 @@ sys32_setxattr_wrapper: lgfr %r6,%r6 # int jg sys_setxattr - .globl sys32_lsetxattr_wrapper -sys32_lsetxattr_wrapper: +ENTRY(sys32_lsetxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * @@ -1123,8 +954,7 @@ sys32_lsetxattr_wrapper: lgfr %r6,%r6 # int jg sys_lsetxattr - .globl sys32_fsetxattr_wrapper -sys32_fsetxattr_wrapper: +ENTRY(sys32_fsetxattr_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * @@ -1132,124 +962,106 @@ sys32_fsetxattr_wrapper: lgfr %r6,%r6 # int jg sys_fsetxattr - .globl sys32_getxattr_wrapper -sys32_getxattr_wrapper: +ENTRY(sys32_getxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * llgfr %r5,%r5 # size_t jg sys_getxattr - .globl sys32_lgetxattr_wrapper -sys32_lgetxattr_wrapper: +ENTRY(sys32_lgetxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * llgfr %r5,%r5 # size_t jg sys_lgetxattr - .globl sys32_fgetxattr_wrapper -sys32_fgetxattr_wrapper: +ENTRY(sys32_fgetxattr_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # char * llgtr %r4,%r4 # void * llgfr %r5,%r5 # size_t jg sys_fgetxattr - .globl sys32_listxattr_wrapper -sys32_listxattr_wrapper: +ENTRY(sys32_listxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t jg sys_listxattr - .globl sys32_llistxattr_wrapper -sys32_llistxattr_wrapper: +ENTRY(sys32_llistxattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t jg sys_llistxattr - .globl sys32_flistxattr_wrapper -sys32_flistxattr_wrapper: +ENTRY(sys32_flistxattr_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t jg sys_flistxattr - .globl sys32_removexattr_wrapper -sys32_removexattr_wrapper: +ENTRY(sys32_removexattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * jg sys_removexattr - .globl sys32_lremovexattr_wrapper -sys32_lremovexattr_wrapper: +ENTRY(sys32_lremovexattr_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # char * jg sys_lremovexattr - .globl sys32_fremovexattr_wrapper -sys32_fremovexattr_wrapper: +ENTRY(sys32_fremovexattr_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # char * jg sys_fremovexattr - .globl sys32_sched_setaffinity_wrapper -sys32_sched_setaffinity_wrapper: +ENTRY(sys32_sched_setaffinity_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # unsigned int llgtr %r4,%r4 # unsigned long * jg compat_sys_sched_setaffinity - .globl sys32_sched_getaffinity_wrapper -sys32_sched_getaffinity_wrapper: +ENTRY(sys32_sched_getaffinity_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # unsigned int llgtr %r4,%r4 # unsigned long * jg compat_sys_sched_getaffinity - .globl sys32_exit_group_wrapper -sys32_exit_group_wrapper: +ENTRY(sys32_exit_group_wrapper) lgfr %r2,%r2 # int jg sys_exit_group # branch to system call - .globl sys32_set_tid_address_wrapper -sys32_set_tid_address_wrapper: +ENTRY(sys32_set_tid_address_wrapper) llgtr %r2,%r2 # int * jg sys_set_tid_address # branch to system call - .globl sys_epoll_create_wrapper -sys_epoll_create_wrapper: +ENTRY(sys_epoll_create_wrapper) lgfr %r2,%r2 # int jg sys_epoll_create # branch to system call - .globl sys_epoll_ctl_wrapper -sys_epoll_ctl_wrapper: +ENTRY(sys_epoll_ctl_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int lgfr %r4,%r4 # int llgtr %r5,%r5 # struct epoll_event * jg sys_epoll_ctl # branch to system call - .globl sys_epoll_wait_wrapper -sys_epoll_wait_wrapper: +ENTRY(sys_epoll_wait_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct epoll_event * lgfr %r4,%r4 # int lgfr %r5,%r5 # int jg sys_epoll_wait # branch to system call - .globl sys32_lookup_dcookie_wrapper -sys32_lookup_dcookie_wrapper: +ENTRY(sys32_lookup_dcookie_wrapper) sllg %r2,%r2,32 # get high word of 64bit dcookie or %r2,%r3 # get low word of 64bit dcookie llgtr %r3,%r4 # char * llgfr %r4,%r5 # size_t jg sys_lookup_dcookie - .globl sys32_fadvise64_wrapper -sys32_fadvise64_wrapper: +ENTRY(sys32_fadvise64_wrapper) lgfr %r2,%r2 # int sllg %r3,%r3,32 # get high word of 64bit loff_t or %r3,%r4 # get low word of 64bit loff_t @@ -1257,81 +1069,68 @@ sys32_fadvise64_wrapper: lgfr %r5,%r6 # int jg sys32_fadvise64 - .globl sys32_fadvise64_64_wrapper -sys32_fadvise64_64_wrapper: +ENTRY(sys32_fadvise64_64_wrapper) llgtr %r2,%r2 # struct fadvise64_64_args * jg sys32_fadvise64_64 - .globl sys32_clock_settime_wrapper -sys32_clock_settime_wrapper: +ENTRY(sys32_clock_settime_wrapper) lgfr %r2,%r2 # clockid_t (int) llgtr %r3,%r3 # struct compat_timespec * jg compat_sys_clock_settime - .globl sys32_clock_gettime_wrapper -sys32_clock_gettime_wrapper: +ENTRY(sys32_clock_gettime_wrapper) lgfr %r2,%r2 # clockid_t (int) llgtr %r3,%r3 # struct compat_timespec * jg compat_sys_clock_gettime - .globl sys32_clock_getres_wrapper -sys32_clock_getres_wrapper: +ENTRY(sys32_clock_getres_wrapper) lgfr %r2,%r2 # clockid_t (int) llgtr %r3,%r3 # struct compat_timespec * jg compat_sys_clock_getres - .globl sys32_clock_nanosleep_wrapper -sys32_clock_nanosleep_wrapper: +ENTRY(sys32_clock_nanosleep_wrapper) lgfr %r2,%r2 # clockid_t (int) lgfr %r3,%r3 # int llgtr %r4,%r4 # struct compat_timespec * llgtr %r5,%r5 # struct compat_timespec * jg compat_sys_clock_nanosleep - .globl sys32_timer_create_wrapper -sys32_timer_create_wrapper: +ENTRY(sys32_timer_create_wrapper) lgfr %r2,%r2 # timer_t (int) llgtr %r3,%r3 # struct compat_sigevent * llgtr %r4,%r4 # timer_t * jg compat_sys_timer_create - .globl sys32_timer_settime_wrapper -sys32_timer_settime_wrapper: +ENTRY(sys32_timer_settime_wrapper) lgfr %r2,%r2 # timer_t (int) lgfr %r3,%r3 # int llgtr %r4,%r4 # struct compat_itimerspec * llgtr %r5,%r5 # struct compat_itimerspec * jg compat_sys_timer_settime - .globl sys32_timer_gettime_wrapper -sys32_timer_gettime_wrapper: +ENTRY(sys32_timer_gettime_wrapper) lgfr %r2,%r2 # timer_t (int) llgtr %r3,%r3 # struct compat_itimerspec * jg compat_sys_timer_gettime - .globl sys32_timer_getoverrun_wrapper -sys32_timer_getoverrun_wrapper: +ENTRY(sys32_timer_getoverrun_wrapper) lgfr %r2,%r2 # timer_t (int) jg sys_timer_getoverrun - .globl sys32_timer_delete_wrapper -sys32_timer_delete_wrapper: +ENTRY(sys32_timer_delete_wrapper) lgfr %r2,%r2 # timer_t (int) jg sys_timer_delete - .globl sys32_io_setup_wrapper -sys32_io_setup_wrapper: +ENTRY(sys32_io_setup_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # u32 * jg compat_sys_io_setup - .globl sys32_io_destroy_wrapper -sys32_io_destroy_wrapper: +ENTRY(sys32_io_destroy_wrapper) llgfr %r2,%r2 # (aio_context_t) u32 jg sys_io_destroy - .globl sys32_io_getevents_wrapper -sys32_io_getevents_wrapper: +ENTRY(sys32_io_getevents_wrapper) llgfr %r2,%r2 # (aio_context_t) u32 lgfr %r3,%r3 # long lgfr %r4,%r4 # long @@ -1339,49 +1138,42 @@ sys32_io_getevents_wrapper: llgtr %r6,%r6 # struct compat_timespec * jg compat_sys_io_getevents - .globl sys32_io_submit_wrapper -sys32_io_submit_wrapper: +ENTRY(sys32_io_submit_wrapper) llgfr %r2,%r2 # (aio_context_t) u32 lgfr %r3,%r3 # long llgtr %r4,%r4 # struct iocb ** jg compat_sys_io_submit - .globl sys32_io_cancel_wrapper -sys32_io_cancel_wrapper: +ENTRY(sys32_io_cancel_wrapper) llgfr %r2,%r2 # (aio_context_t) u32 llgtr %r3,%r3 # struct iocb * llgtr %r4,%r4 # struct io_event * jg sys_io_cancel - .globl compat_sys_statfs64_wrapper -compat_sys_statfs64_wrapper: +ENTRY(compat_sys_statfs64_wrapper) llgtr %r2,%r2 # const char * llgfr %r3,%r3 # compat_size_t llgtr %r4,%r4 # struct compat_statfs64 * jg compat_sys_statfs64 - .globl compat_sys_fstatfs64_wrapper -compat_sys_fstatfs64_wrapper: +ENTRY(compat_sys_fstatfs64_wrapper) llgfr %r2,%r2 # unsigned int fd llgfr %r3,%r3 # compat_size_t llgtr %r4,%r4 # struct compat_statfs64 * jg compat_sys_fstatfs64 - .globl compat_sys_mq_open_wrapper -compat_sys_mq_open_wrapper: +ENTRY(compat_sys_mq_open_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int llgfr %r4,%r4 # mode_t llgtr %r5,%r5 # struct compat_mq_attr * jg compat_sys_mq_open - .globl sys32_mq_unlink_wrapper -sys32_mq_unlink_wrapper: +ENTRY(sys32_mq_unlink_wrapper) llgtr %r2,%r2 # const char * jg sys_mq_unlink - .globl compat_sys_mq_timedsend_wrapper -compat_sys_mq_timedsend_wrapper: +ENTRY(compat_sys_mq_timedsend_wrapper) lgfr %r2,%r2 # mqd_t llgtr %r3,%r3 # const char * llgfr %r4,%r4 # size_t @@ -1389,8 +1181,7 @@ compat_sys_mq_timedsend_wrapper: llgtr %r6,%r6 # const struct compat_timespec * jg compat_sys_mq_timedsend - .globl compat_sys_mq_timedreceive_wrapper -compat_sys_mq_timedreceive_wrapper: +ENTRY(compat_sys_mq_timedreceive_wrapper) lgfr %r2,%r2 # mqd_t llgtr %r3,%r3 # char * llgfr %r4,%r4 # size_t @@ -1398,21 +1189,18 @@ compat_sys_mq_timedreceive_wrapper: llgtr %r6,%r6 # const struct compat_timespec * jg compat_sys_mq_timedreceive - .globl compat_sys_mq_notify_wrapper -compat_sys_mq_notify_wrapper: +ENTRY(compat_sys_mq_notify_wrapper) lgfr %r2,%r2 # mqd_t llgtr %r3,%r3 # struct compat_sigevent * jg compat_sys_mq_notify - .globl compat_sys_mq_getsetattr_wrapper -compat_sys_mq_getsetattr_wrapper: +ENTRY(compat_sys_mq_getsetattr_wrapper) lgfr %r2,%r2 # mqd_t llgtr %r3,%r3 # struct compat_mq_attr * llgtr %r4,%r4 # struct compat_mq_attr * jg compat_sys_mq_getsetattr - .globl compat_sys_add_key_wrapper -compat_sys_add_key_wrapper: +ENTRY(compat_sys_add_key_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * llgtr %r4,%r4 # const void * @@ -1420,16 +1208,14 @@ compat_sys_add_key_wrapper: llgfr %r6,%r6 # (key_serial_t) u32 jg sys_add_key - .globl compat_sys_request_key_wrapper -compat_sys_request_key_wrapper: +ENTRY(compat_sys_request_key_wrapper) llgtr %r2,%r2 # const char * llgtr %r3,%r3 # const char * llgtr %r4,%r4 # const void * llgfr %r5,%r5 # (key_serial_t) u32 jg sys_request_key - .globl sys32_remap_file_pages_wrapper -sys32_remap_file_pages_wrapper: +ENTRY(sys32_remap_file_pages_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # unsigned long llgfr %r4,%r4 # unsigned long @@ -1437,8 +1223,7 @@ sys32_remap_file_pages_wrapper: llgfr %r6,%r6 # unsigned long jg sys_remap_file_pages - .globl compat_sys_waitid_wrapper -compat_sys_waitid_wrapper: +ENTRY(compat_sys_waitid_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # pid_t llgtr %r4,%r4 # siginfo_emu31_t * @@ -1446,65 +1231,56 @@ compat_sys_waitid_wrapper: llgtr %r6,%r6 # struct rusage_emu31 * jg compat_sys_waitid - .globl compat_sys_kexec_load_wrapper -compat_sys_kexec_load_wrapper: +ENTRY(compat_sys_kexec_load_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # unsigned long llgtr %r4,%r4 # struct kexec_segment * llgfr %r5,%r5 # unsigned long jg compat_sys_kexec_load - .globl sys_ioprio_set_wrapper -sys_ioprio_set_wrapper: +ENTRY(sys_ioprio_set_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int lgfr %r4,%r4 # int jg sys_ioprio_set - .globl sys_ioprio_get_wrapper -sys_ioprio_get_wrapper: +ENTRY(sys_ioprio_get_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int jg sys_ioprio_get - .globl sys_inotify_add_watch_wrapper -sys_inotify_add_watch_wrapper: +ENTRY(sys_inotify_add_watch_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # u32 jg sys_inotify_add_watch - .globl sys_inotify_rm_watch_wrapper -sys_inotify_rm_watch_wrapper: +ENTRY(sys_inotify_rm_watch_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # u32 jg sys_inotify_rm_watch - .globl compat_sys_openat_wrapper -compat_sys_openat_wrapper: +ENTRY(compat_sys_openat_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int lgfr %r5,%r5 # int jg compat_sys_openat - .globl sys_mkdirat_wrapper -sys_mkdirat_wrapper: +ENTRY(sys_mkdirat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int jg sys_mkdirat - .globl sys_mknodat_wrapper -sys_mknodat_wrapper: +ENTRY(sys_mknodat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int llgfr %r5,%r5 # unsigned int jg sys_mknodat - .globl sys_fchownat_wrapper -sys_fchownat_wrapper: +ENTRY(sys_fchownat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # uid_t @@ -1512,38 +1288,33 @@ sys_fchownat_wrapper: lgfr %r6,%r6 # int jg sys_fchownat - .globl compat_sys_futimesat_wrapper -compat_sys_futimesat_wrapper: +ENTRY(compat_sys_futimesat_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # char * llgtr %r4,%r4 # struct timeval * jg compat_sys_futimesat - .globl sys32_fstatat64_wrapper -sys32_fstatat64_wrapper: +ENTRY(sys32_fstatat64_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # char * llgtr %r4,%r4 # struct stat64 * lgfr %r5,%r5 # int jg sys32_fstatat64 - .globl sys_unlinkat_wrapper -sys_unlinkat_wrapper: +ENTRY(sys_unlinkat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int jg sys_unlinkat - .globl sys_renameat_wrapper -sys_renameat_wrapper: +ENTRY(sys_renameat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int llgtr %r5,%r5 # const char * jg sys_renameat - .globl sys_linkat_wrapper -sys_linkat_wrapper: +ENTRY(sys_linkat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int @@ -1551,37 +1322,32 @@ sys_linkat_wrapper: lgfr %r6,%r6 # int jg sys_linkat - .globl sys_symlinkat_wrapper -sys_symlinkat_wrapper: +ENTRY(sys_symlinkat_wrapper) llgtr %r2,%r2 # const char * lgfr %r3,%r3 # int llgtr %r4,%r4 # const char * jg sys_symlinkat - .globl sys_readlinkat_wrapper -sys_readlinkat_wrapper: +ENTRY(sys_readlinkat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * llgtr %r4,%r4 # char * lgfr %r5,%r5 # int jg sys_readlinkat - .globl sys_fchmodat_wrapper -sys_fchmodat_wrapper: +ENTRY(sys_fchmodat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * llgfr %r4,%r4 # mode_t jg sys_fchmodat - .globl sys_faccessat_wrapper -sys_faccessat_wrapper: +ENTRY(sys_faccessat_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char * lgfr %r4,%r4 # int jg sys_faccessat - .globl compat_sys_pselect6_wrapper -compat_sys_pselect6_wrapper: +ENTRY(compat_sys_pselect6_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # fd_set * llgtr %r4,%r4 # fd_set * @@ -1591,8 +1357,7 @@ compat_sys_pselect6_wrapper: stg %r0,160(%r15) jg compat_sys_pselect6 - .globl compat_sys_ppoll_wrapper -compat_sys_ppoll_wrapper: +ENTRY(compat_sys_ppoll_wrapper) llgtr %r2,%r2 # struct pollfd * llgfr %r3,%r3 # unsigned int llgtr %r4,%r4 # struct timespec * @@ -1600,26 +1365,22 @@ compat_sys_ppoll_wrapper: llgfr %r6,%r6 # size_t jg compat_sys_ppoll - .globl sys_unshare_wrapper -sys_unshare_wrapper: +ENTRY(sys_unshare_wrapper) llgfr %r2,%r2 # unsigned long jg sys_unshare - .globl compat_sys_set_robust_list_wrapper -compat_sys_set_robust_list_wrapper: +ENTRY(compat_sys_set_robust_list_wrapper) llgtr %r2,%r2 # struct compat_robust_list_head * llgfr %r3,%r3 # size_t jg compat_sys_set_robust_list - .globl compat_sys_get_robust_list_wrapper -compat_sys_get_robust_list_wrapper: +ENTRY(compat_sys_get_robust_list_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_uptr_t_t * llgtr %r4,%r4 # compat_size_t * jg compat_sys_get_robust_list - .globl sys_splice_wrapper -sys_splice_wrapper: +ENTRY(sys_splice_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # loff_t * lgfr %r4,%r4 # int @@ -1629,8 +1390,7 @@ sys_splice_wrapper: stg %r0,160(%r15) jg sys_splice - .globl sys_sync_file_range_wrapper -sys_sync_file_range_wrapper: +ENTRY(sys_sync_file_range_wrapper) lgfr %r2,%r2 # int sllg %r3,%r3,32 # get high word of 64bit loff_t or %r3,%r4 # get low word of 64bit loff_t @@ -1639,31 +1399,27 @@ sys_sync_file_range_wrapper: llgf %r5,164(%r15) # unsigned int jg sys_sync_file_range - .globl sys_tee_wrapper -sys_tee_wrapper: +ENTRY(sys_tee_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgfr %r4,%r4 # size_t llgfr %r5,%r5 # unsigned int jg sys_tee - .globl compat_sys_vmsplice_wrapper -compat_sys_vmsplice_wrapper: +ENTRY(compat_sys_vmsplice_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_iovec * llgfr %r4,%r4 # unsigned int llgfr %r5,%r5 # unsigned int jg compat_sys_vmsplice - .globl sys_getcpu_wrapper -sys_getcpu_wrapper: +ENTRY(sys_getcpu_wrapper) llgtr %r2,%r2 # unsigned * llgtr %r3,%r3 # unsigned * llgtr %r4,%r4 # struct getcpu_cache * jg sys_getcpu - .globl compat_sys_epoll_pwait_wrapper -compat_sys_epoll_pwait_wrapper: +ENTRY(compat_sys_epoll_pwait_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct compat_epoll_event * lgfr %r4,%r4 # int @@ -1673,34 +1429,29 @@ compat_sys_epoll_pwait_wrapper: stg %r0,160(%r15) jg compat_sys_epoll_pwait - .globl compat_sys_utimes_wrapper -compat_sys_utimes_wrapper: +ENTRY(compat_sys_utimes_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # struct compat_timeval * jg compat_sys_utimes - .globl compat_sys_utimensat_wrapper -compat_sys_utimensat_wrapper: +ENTRY(compat_sys_utimensat_wrapper) llgfr %r2,%r2 # unsigned int llgtr %r3,%r3 # char * llgtr %r4,%r4 # struct compat_timespec * lgfr %r5,%r5 # int jg compat_sys_utimensat - .globl compat_sys_signalfd_wrapper -compat_sys_signalfd_wrapper: +ENTRY(compat_sys_signalfd_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_sigset_t * llgfr %r4,%r4 # compat_size_t jg compat_sys_signalfd - .globl sys_eventfd_wrapper -sys_eventfd_wrapper: +ENTRY(sys_eventfd_wrapper) llgfr %r2,%r2 # unsigned int jg sys_eventfd - .globl sys_fallocate_wrapper -sys_fallocate_wrapper: +ENTRY(sys_fallocate_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int sllg %r4,%r4,32 # get high word of 64bit loff_t @@ -1709,94 +1460,80 @@ sys_fallocate_wrapper: l %r5,164(%r15) # get low word of 64bit loff_t jg sys_fallocate - .globl sys_timerfd_create_wrapper -sys_timerfd_create_wrapper: +ENTRY(sys_timerfd_create_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int jg sys_timerfd_create - .globl compat_sys_timerfd_settime_wrapper -compat_sys_timerfd_settime_wrapper: +ENTRY(compat_sys_timerfd_settime_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgtr %r4,%r4 # struct compat_itimerspec * llgtr %r5,%r5 # struct compat_itimerspec * jg compat_sys_timerfd_settime - .globl compat_sys_timerfd_gettime_wrapper -compat_sys_timerfd_gettime_wrapper: +ENTRY(compat_sys_timerfd_gettime_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct compat_itimerspec * jg compat_sys_timerfd_gettime - .globl compat_sys_signalfd4_wrapper -compat_sys_signalfd4_wrapper: +ENTRY(compat_sys_signalfd4_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # compat_sigset_t * llgfr %r4,%r4 # compat_size_t lgfr %r5,%r5 # int jg compat_sys_signalfd4 - .globl sys_eventfd2_wrapper -sys_eventfd2_wrapper: +ENTRY(sys_eventfd2_wrapper) llgfr %r2,%r2 # unsigned int lgfr %r3,%r3 # int jg sys_eventfd2 - .globl sys_inotify_init1_wrapper -sys_inotify_init1_wrapper: +ENTRY(sys_inotify_init1_wrapper) lgfr %r2,%r2 # int jg sys_inotify_init1 - .globl sys_pipe2_wrapper -sys_pipe2_wrapper: +ENTRY(sys_pipe2_wrapper) llgtr %r2,%r2 # u32 * lgfr %r3,%r3 # int jg sys_pipe2 # branch to system call - .globl sys_dup3_wrapper -sys_dup3_wrapper: +ENTRY(sys_dup3_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int lgfr %r4,%r4 # int jg sys_dup3 # branch to system call - .globl sys_epoll_create1_wrapper -sys_epoll_create1_wrapper: +ENTRY(sys_epoll_create1_wrapper) lgfr %r2,%r2 # int jg sys_epoll_create1 # branch to system call - .globl sys32_readahead_wrapper -sys32_readahead_wrapper: +ENTRY(sys32_readahead_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # u32 llgfr %r4,%r4 # u32 lgfr %r5,%r5 # s32 jg sys32_readahead # branch to system call - .globl sys32_sendfile64_wrapper -sys32_sendfile64_wrapper: +ENTRY(sys32_sendfile64_wrapper) lgfr %r2,%r2 # int lgfr %r3,%r3 # int llgtr %r4,%r4 # compat_loff_t * lgfr %r5,%r5 # s32 jg sys32_sendfile64 # branch to system call - .globl sys_tkill_wrapper -sys_tkill_wrapper: +ENTRY(sys_tkill_wrapper) lgfr %r2,%r2 # pid_t lgfr %r3,%r3 # int jg sys_tkill # branch to system call - .globl sys_tgkill_wrapper -sys_tgkill_wrapper: +ENTRY(sys_tgkill_wrapper) lgfr %r2,%r2 # pid_t lgfr %r3,%r3 # pid_t lgfr %r4,%r4 # int jg sys_tgkill # branch to system call - .globl compat_sys_keyctl_wrapper -compat_sys_keyctl_wrapper: +ENTRY(compat_sys_keyctl_wrapper) llgfr %r2,%r2 # u32 llgfr %r3,%r3 # u32 llgfr %r4,%r4 # u32 @@ -1804,8 +1541,7 @@ compat_sys_keyctl_wrapper: llgfr %r6,%r6 # u32 jg compat_sys_keyctl # branch to system call - .globl compat_sys_preadv_wrapper -compat_sys_preadv_wrapper: +ENTRY(compat_sys_preadv_wrapper) llgfr %r2,%r2 # unsigned long llgtr %r3,%r3 # compat_iovec * llgfr %r4,%r4 # unsigned long @@ -1813,8 +1549,7 @@ compat_sys_preadv_wrapper: llgfr %r6,%r6 # u32 jg compat_sys_preadv # branch to system call - .globl compat_sys_pwritev_wrapper -compat_sys_pwritev_wrapper: +ENTRY(compat_sys_pwritev_wrapper) llgfr %r2,%r2 # unsigned long llgtr %r3,%r3 # compat_iovec * llgfr %r4,%r4 # unsigned long @@ -1822,16 +1557,14 @@ compat_sys_pwritev_wrapper: llgfr %r6,%r6 # u32 jg compat_sys_pwritev # branch to system call - .globl compat_sys_rt_tgsigqueueinfo_wrapper -compat_sys_rt_tgsigqueueinfo_wrapper: +ENTRY(compat_sys_rt_tgsigqueueinfo_wrapper) lgfr %r2,%r2 # compat_pid_t lgfr %r3,%r3 # compat_pid_t lgfr %r4,%r4 # int llgtr %r5,%r5 # struct compat_siginfo * jg compat_sys_rt_tgsigqueueinfo_wrapper # branch to system call - .globl sys_perf_event_open_wrapper -sys_perf_event_open_wrapper: +ENTRY(sys_perf_event_open_wrapper) llgtr %r2,%r2 # const struct perf_event_attr * lgfr %r3,%r3 # pid_t lgfr %r4,%r4 # int @@ -1839,29 +1572,25 @@ sys_perf_event_open_wrapper: llgfr %r6,%r6 # unsigned long jg sys_perf_event_open # branch to system call - .globl sys_clone_wrapper -sys_clone_wrapper: +ENTRY(sys_clone_wrapper) llgfr %r2,%r2 # unsigned long llgfr %r3,%r3 # unsigned long llgtr %r4,%r4 # int * llgtr %r5,%r5 # int * jg sys_clone # branch to system call - .globl sys32_execve_wrapper -sys32_execve_wrapper: +ENTRY(sys32_execve_wrapper) llgtr %r2,%r2 # char * llgtr %r3,%r3 # compat_uptr_t * llgtr %r4,%r4 # compat_uptr_t * jg sys32_execve # branch to system call - .globl sys_fanotify_init_wrapper -sys_fanotify_init_wrapper: +ENTRY(sys_fanotify_init_wrapper) llgfr %r2,%r2 # unsigned int llgfr %r3,%r3 # unsigned int jg sys_fanotify_init # branch to system call - .globl sys_fanotify_mark_wrapper -sys_fanotify_mark_wrapper: +ENTRY(sys_fanotify_mark_wrapper) lgfr %r2,%r2 # int llgfr %r3,%r3 # unsigned int sllg %r4,%r4,32 # get high word of 64bit mask @@ -1870,16 +1599,14 @@ sys_fanotify_mark_wrapper: llgt %r6,164(%r15) # char * jg sys_fanotify_mark # branch to system call - .globl sys_prlimit64_wrapper -sys_prlimit64_wrapper: +ENTRY(sys_prlimit64_wrapper) lgfr %r2,%r2 # pid_t llgfr %r3,%r3 # unsigned int llgtr %r4,%r4 # const struct rlimit64 __user * llgtr %r5,%r5 # struct rlimit64 __user * jg sys_prlimit64 # branch to system call - .globl sys_name_to_handle_at_wrapper -sys_name_to_handle_at_wrapper: +ENTRY(sys_name_to_handle_at_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # const char __user * llgtr %r4,%r4 # struct file_handle __user * @@ -1887,20 +1614,23 @@ sys_name_to_handle_at_wrapper: lgfr %r6,%r6 # int jg sys_name_to_handle_at - .globl compat_sys_open_by_handle_at_wrapper -compat_sys_open_by_handle_at_wrapper: +ENTRY(compat_sys_open_by_handle_at_wrapper) lgfr %r2,%r2 # int llgtr %r3,%r3 # struct file_handle __user * lgfr %r4,%r4 # int jg compat_sys_open_by_handle_at - .globl compat_sys_clock_adjtime_wrapper -compat_sys_clock_adjtime_wrapper: +ENTRY(compat_sys_clock_adjtime_wrapper) lgfr %r2,%r2 # clockid_t (int) llgtr %r3,%r3 # struct compat_timex __user * jg compat_sys_clock_adjtime - .globl sys_syncfs_wrapper -sys_syncfs_wrapper: +ENTRY(sys_syncfs_wrapper) lgfr %r2,%r2 # int jg sys_syncfs + + .globl sys_setns_wrapper +sys_setns_wrapper: + lgfr %r2,%r2 # int + lgfr %r3,%r3 # int + jg sys_setns diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index c032d11da8a1..8237fc07ac79 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -9,27 +9,6 @@ #include <asm/diag.h> /* - * Diagnose 10: Release pages - */ -void diag10(unsigned long addr) -{ - if (addr >= 0x7ff00000) - return; - asm volatile( -#ifdef CONFIG_64BIT - " sam31\n" - " diag %0,%0,0x10\n" - "0: sam64\n" -#else - " diag %0,%0,0x10\n" - "0:\n" -#endif - EX_TABLE(0b, 0b) - : : "a" (addr)); -} -EXPORT_SYMBOL(diag10); - -/* * Diagnose 14: Input spool file manipulation */ int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index c83726c9fe03..45df6d456aa1 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -27,12 +27,12 @@ #include <asm/system.h> #include <asm/uaccess.h> #include <asm/io.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/mathemu.h> #include <asm/cpcmd.h> -#include <asm/s390_ext.h> #include <asm/lowcore.h> #include <asm/debug.h> +#include <asm/irq.h> #ifndef CONFIG_64BIT #define ONELONG "%08lx: " @@ -672,6 +672,7 @@ static struct insn opcode_b2[] = { { "rp", 0x77, INSTR_S_RD }, { "stcke", 0x78, INSTR_S_RD }, { "sacf", 0x79, INSTR_S_RD }, + { "spp", 0x80, INSTR_S_RD }, { "stsi", 0x7d, INSTR_S_RD }, { "srnm", 0x99, INSTR_S_RD }, { "stfpc", 0x9c, INSTR_S_RD }, diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 648f64239a9d..02ec8fe7d03f 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -9,8 +9,8 @@ * Heiko Carstens <heiko.carstens@de.ibm.com> */ -#include <linux/linkage.h> #include <linux/init.h> +#include <linux/linkage.h> #include <asm/cache.h> #include <asm/errno.h> #include <asm/ptrace.h> @@ -197,8 +197,7 @@ STACK_SIZE = 1 << STACK_SHIFT * Returns: * gpr2 = prev */ - .globl __switch_to -__switch_to: +ENTRY(__switch_to) basr %r1,0 0: l %r4,__THREAD_info(%r2) # get thread_info of prev l %r5,__THREAD_info(%r3) # get thread_info of next @@ -212,6 +211,7 @@ __switch_to: lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task st %r3,__LC_CURRENT # store task struct of next + mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next st %r5,__LC_THREAD_INFO # store thread info of next ahi %r5,STACK_SIZE # end of kernel stack of next st %r5,__LC_KERNEL_STACK # store end of kernel stack @@ -223,8 +223,7 @@ __critical_start: * are executed with interrupts enabled. */ - .globl system_call -system_call: +ENTRY(system_call) stpt __LC_SYNC_ENTER_TIMER sysc_saveall: SAVE_ALL_SVC __LC_SVC_OLD_PSW,__LC_SAVE_AREA @@ -387,8 +386,7 @@ sysc_tracenogo: # # a new process exits the kernel with ret_from_fork # - .globl ret_from_fork -ret_from_fork: +ENTRY(ret_from_fork) l %r13,__LC_SVC_NEW_PSW+4 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct tm SP_PSW+1(%r15),0x01 # forking a kernel thread ? @@ -404,8 +402,7 @@ ret_from_fork: # kernel_execve function needs to deal with pt_regs that is not # at the usual place # - .globl kernel_execve -kernel_execve: +ENTRY(kernel_execve) stm %r12,%r15,48(%r15) lr %r14,%r15 l %r13,__LC_SVC_NEW_PSW+4 @@ -437,8 +434,7 @@ kernel_execve: * Program check handler routine */ - .globl pgm_check_handler -pgm_check_handler: +ENTRY(pgm_check_handler) /* * First we need to check for a special case: * Single stepping an instruction that disables the PER event mask will @@ -564,8 +560,7 @@ kernel_per: * IO interrupt handler routine */ - .globl io_int_handler -io_int_handler: +ENTRY(io_int_handler) stck __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+16 @@ -702,8 +697,7 @@ io_notify_resume: * External interrupt handler routine */ - .globl ext_int_handler -ext_int_handler: +ENTRY(ext_int_handler) stck __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+16 @@ -730,8 +724,7 @@ __critical_end: * Machine check handler routines */ - .globl mcck_int_handler -mcck_int_handler: +ENTRY(mcck_int_handler) stck __LC_MCCK_CLOCK spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs @@ -817,8 +810,7 @@ mcck_return: */ #ifdef CONFIG_SMP __CPUINIT - .globl restart_int_handler -restart_int_handler: +ENTRY(restart_int_handler) basr %r1,0 restart_base: spt restart_vtime-restart_base(%r1) @@ -836,7 +828,7 @@ restart_base: stosm __SF_EMPTY(%r15),0x04 # now we can turn dat on basr %r14,0 l %r14,restart_addr-.(%r14) - br %r14 # branch to start_secondary + basr %r14,%r14 # branch to start_secondary restart_addr: .long start_secondary .align 8 @@ -847,8 +839,7 @@ restart_vtime: /* * If we do not run with SMP enabled, let the new CPU crash ... */ - .globl restart_int_handler -restart_int_handler: +ENTRY(restart_int_handler) basr %r1,0 restart_base: lpsw restart_crash-restart_base(%r1) @@ -858,6 +849,34 @@ restart_crash: restart_go: #endif +# +# PSW restart interrupt handler +# +ENTRY(psw_restart_int_handler) + st %r15,__LC_SAVE_AREA_64(%r0) # save r15 + basr %r15,0 +0: l %r15,.Lrestart_stack-0b(%r15) # load restart stack + l %r15,0(%r15) + ahi %r15,-SP_SIZE # make room for pt_regs + stm %r0,%r14,SP_R0(%r15) # store gprs %r0-%r14 to stack + mvc SP_R15(4,%r15),__LC_SAVE_AREA_64(%r0)# store saved %r15 to stack + mvc SP_PSW(8,%r15),__LC_RST_OLD_PSW(%r0) # store restart old psw + xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # set backchain to 0 + basr %r14,0 +1: l %r14,.Ldo_restart-1b(%r14) + basr %r14,%r14 + + basr %r14,0 # load disabled wait PSW if +2: lpsw restart_psw_crash-2b(%r14) # do_restart returns + .align 4 +.Ldo_restart: + .long do_restart +.Lrestart_stack: + .long restart_stack + .align 8 +restart_psw_crash: + .long 0x000a0000,0x00000000 + restart_psw_crash + .section .kprobes.text, "ax" #ifdef CONFIG_CHECK_STACK diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 17a6f83a2d67..66729eb7bbc5 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -5,10 +5,9 @@ #include <linux/signal.h> #include <asm/ptrace.h> -typedef void pgm_check_handler_t(struct pt_regs *, long, unsigned long); -extern pgm_check_handler_t *pgm_check_table[128]; -pgm_check_handler_t do_protection_exception; -pgm_check_handler_t do_dat_exception; +void do_protection_exception(struct pt_regs *, long, unsigned long); +void do_dat_exception(struct pt_regs *, long, unsigned long); +void do_asce_exception(struct pt_regs *, long, unsigned long); extern int sysctl_userprocess_debug; diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 9d3603d6c511..5f729d627cef 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -9,8 +9,8 @@ * Heiko Carstens <heiko.carstens@de.ibm.com> */ -#include <linux/linkage.h> #include <linux/init.h> +#include <linux/linkage.h> #include <asm/cache.h> #include <asm/errno.h> #include <asm/ptrace.h> @@ -56,15 +56,28 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_MCCK_PENDING) _TIF_SYSCALL = (_TIF_SYSCALL_TRACE>>8 | _TIF_SYSCALL_AUDIT>>8 | \ _TIF_SECCOMP>>8 | _TIF_SYSCALL_TRACEPOINT>>8) +_TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING) #define BASED(name) name-system_call(%r13) + .macro SPP newpp +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) + tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_SPP + jz .+8 + .insn s,0xb2800000,\newpp +#endif + .endm + .macro HANDLE_SIE_INTERCEPT #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) - lg %r3,__LC_SIE_HOOK - ltgr %r3,%r3 + tm __TI_flags+6(%r12),_TIF_SIE>>8 jz 0f - basr %r14,%r3 + SPP __LC_CMF_HPP # set host id + clc SP_PSW+8(8,%r15),BASED(.Lsie_loop) + jl 0f + clc SP_PSW+8(8,%r15),BASED(.Lsie_done) + jhe 0f + mvc SP_PSW+8(8,%r15),BASED(.Lsie_loop) 0: #endif .endm @@ -206,8 +219,7 @@ _TIF_SYSCALL = (_TIF_SYSCALL_TRACE>>8 | _TIF_SYSCALL_AUDIT>>8 | \ * Returns: * gpr2 = prev */ - .globl __switch_to -__switch_to: +ENTRY(__switch_to) lg %r4,__THREAD_info(%r2) # get thread_info of prev lg %r5,__THREAD_info(%r3) # get thread_info of next tm __TI_flags+7(%r4),_TIF_MCCK_PENDING # machine check pending? @@ -220,6 +232,7 @@ __switch_to: lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task stg %r3,__LC_CURRENT # store task struct of next + mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next stg %r5,__LC_THREAD_INFO # store thread info of next aghi %r5,STACK_SIZE # end of kernel stack of next stg %r5,__LC_KERNEL_STACK # store end of kernel stack @@ -231,8 +244,7 @@ __critical_start: * are executed with interrupts enabled. */ - .globl system_call -system_call: +ENTRY(system_call) stpt __LC_SYNC_ENTER_TIMER sysc_saveall: SAVE_ALL_SVC __LC_SVC_OLD_PSW,__LC_SAVE_AREA @@ -394,8 +406,7 @@ sysc_tracenogo: # # a new process exits the kernel with ret_from_fork # - .globl ret_from_fork -ret_from_fork: +ENTRY(ret_from_fork) lg %r13,__LC_SVC_NEW_PSW+8 lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct tm SP_PSW+1(%r15),0x01 # forking a kernel thread ? @@ -410,8 +421,7 @@ ret_from_fork: # kernel_execve function needs to deal with pt_regs that is not # at the usual place # - .globl kernel_execve -kernel_execve: +ENTRY(kernel_execve) stmg %r12,%r15,96(%r15) lgr %r14,%r15 aghi %r15,-SP_SIZE @@ -441,8 +451,7 @@ kernel_execve: * Program check handler routine */ - .globl pgm_check_handler -pgm_check_handler: +ENTRY(pgm_check_handler) /* * First we need to check for a special case: * Single stepping an instruction that disables the PER event mask will @@ -464,6 +473,7 @@ pgm_check_handler: xc SP_ILC(4,%r15),SP_ILC(%r15) mvc SP_PSW(16,%r15),__LC_PGM_OLD_PSW lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct + HANDLE_SIE_INTERCEPT tm SP_PSW+1(%r15),0x01 # interrupting from user ? jz pgm_no_vtime UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER @@ -471,7 +481,6 @@ pgm_check_handler: mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER LAST_BREAK pgm_no_vtime: - HANDLE_SIE_INTERCEPT stg %r11,SP_ARGS(%r15) lgf %r3,__LC_PGM_ILC # load program interruption code lg %r4,__LC_TRANS_EXC_CODE @@ -506,6 +515,7 @@ pgm_per_std: CREATE_STACK_FRAME __LC_SAVE_AREA mvc SP_PSW(16,%r15),__LC_PGM_OLD_PSW lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct + HANDLE_SIE_INTERCEPT tm SP_PSW+1(%r15),0x01 # interrupting from user ? jz pgm_no_vtime2 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER @@ -513,7 +523,6 @@ pgm_per_std: mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER LAST_BREAK pgm_no_vtime2: - HANDLE_SIE_INTERCEPT lg %r1,__TI_task(%r12) tm SP_PSW+1(%r15),0x01 # kernel per event ? jz kernel_per @@ -570,14 +579,14 @@ kernel_per: /* * IO interrupt handler routine */ - .globl io_int_handler -io_int_handler: +ENTRY(io_int_handler) stck __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+40 CREATE_STACK_FRAME __LC_SAVE_AREA+40 mvc SP_PSW(16,%r15),0(%r12) # move user PSW to stack lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct + HANDLE_SIE_INTERCEPT tm SP_PSW+1(%r15),0x01 # interrupting from user ? jz io_no_vtime UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER @@ -585,7 +594,6 @@ io_int_handler: mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER LAST_BREAK io_no_vtime: - HANDLE_SIE_INTERCEPT TRACE_IRQS_OFF la %r2,SP_PTREGS(%r15) # address of register-save area brasl %r14,do_IRQ # call standard irq handler @@ -705,14 +713,14 @@ io_notify_resume: /* * External interrupt handler routine */ - .globl ext_int_handler -ext_int_handler: +ENTRY(ext_int_handler) stck __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+40 CREATE_STACK_FRAME __LC_SAVE_AREA+40 mvc SP_PSW(16,%r15),0(%r12) # move user PSW to stack lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct + HANDLE_SIE_INTERCEPT tm SP_PSW+1(%r15),0x01 # interrupting from user ? jz ext_no_vtime UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER @@ -720,7 +728,6 @@ ext_int_handler: mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER LAST_BREAK ext_no_vtime: - HANDLE_SIE_INTERCEPT TRACE_IRQS_OFF lghi %r1,4096 la %r2,SP_PTREGS(%r15) # address of register-save area @@ -735,8 +742,7 @@ __critical_end: /* * Machine check handler routines */ - .globl mcck_int_handler -mcck_int_handler: +ENTRY(mcck_int_handler) stck __LC_MCCK_CLOCK la %r1,4095 # revalidate r1 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer @@ -784,6 +790,7 @@ mcck_int_main: lg %r12,__LC_THREAD_INFO # load pointer to thread_info struct tm __LC_MCCK_CODE+2,0x08 # mwp of old psw valid? jno mcck_no_vtime # no -> no timer update + HANDLE_SIE_INTERCEPT tm SP_PSW+1(%r15),0x01 # interrupting from user ? jz mcck_no_vtime UPDATE_VTIME __LC_EXIT_TIMER,__LC_MCCK_ENTER_TIMER,__LC_USER_TIMER @@ -803,7 +810,6 @@ mcck_no_vtime: stosm __SF_EMPTY(%r15),0x04 # turn dat on tm __TI_flags+7(%r12),_TIF_MCCK_PENDING jno mcck_return - HANDLE_SIE_INTERCEPT TRACE_IRQS_OFF brasl %r14,s390_handle_mcck TRACE_IRQS_ON @@ -822,8 +828,7 @@ mcck_done: */ #ifdef CONFIG_SMP __CPUINIT - .globl restart_int_handler -restart_int_handler: +ENTRY(restart_int_handler) basr %r1,0 restart_base: spt restart_vtime-restart_base(%r1) @@ -841,7 +846,7 @@ restart_base: mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1) xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER stosm __SF_EMPTY(%r15),0x04 # now we can turn dat on - jg start_secondary + brasl %r14,start_secondary .align 8 restart_vtime: .long 0x7fffffff,0xffffffff @@ -850,8 +855,7 @@ restart_vtime: /* * If we do not run with SMP enabled, let the new CPU crash ... */ - .globl restart_int_handler -restart_int_handler: +ENTRY(restart_int_handler) basr %r1,0 restart_base: lpswe restart_crash-restart_base(%r1) @@ -861,6 +865,26 @@ restart_crash: restart_go: #endif +# +# PSW restart interrupt handler +# +ENTRY(psw_restart_int_handler) + stg %r15,__LC_SAVE_AREA_64(%r0) # save r15 + larl %r15,restart_stack # load restart stack + lg %r15,0(%r15) + aghi %r15,-SP_SIZE # make room for pt_regs + stmg %r0,%r14,SP_R0(%r15) # store gprs %r0-%r14 to stack + mvc SP_R15(8,%r15),__LC_SAVE_AREA_64(%r0)# store saved %r15 to stack + mvc SP_PSW(16,%r15),__LC_RST_OLD_PSW(%r0)# store restart old psw + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) # set backchain to 0 + brasl %r14,do_restart + + larl %r14,restart_psw_crash # load disabled wait PSW if + lpswe 0(%r14) # do_restart returns + .align 8 +restart_psw_crash: + .quad 0x0002000080000000,0x0000000000000000 + restart_psw_crash + .section .kprobes.text, "ax" #ifdef CONFIG_CHECK_STACK @@ -1035,6 +1059,56 @@ cleanup_io_restore_insn: .Lcritical_end: .quad __critical_end +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +/* + * sie64a calling convention: + * %r2 pointer to sie control block + * %r3 guest register save area + */ +ENTRY(sie64a) + stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers + stg %r2,__SF_EMPTY(%r15) # save control block pointer + stg %r3,__SF_EMPTY+8(%r15) # save guest register save area + lmg %r0,%r13,0(%r3) # load guest gprs 0-13 + lg %r14,__LC_THREAD_INFO # pointer thread_info struct + oi __TI_flags+6(%r14),_TIF_SIE>>8 +sie_loop: + lg %r14,__LC_THREAD_INFO # pointer thread_info struct + tm __TI_flags+7(%r14),_TIF_EXIT_SIE + jnz sie_exit + lg %r14,__SF_EMPTY(%r15) # get control block pointer + SPP __SF_EMPTY(%r15) # set guest id + sie 0(%r14) +sie_done: + SPP __LC_CMF_HPP # set host id + lg %r14,__LC_THREAD_INFO # pointer thread_info struct +sie_exit: + ni __TI_flags+6(%r14),255-(_TIF_SIE>>8) + lg %r14,__SF_EMPTY+8(%r15) # load guest register save area + stmg %r0,%r13,0(%r14) # save guest gprs 0-13 + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + lghi %r2,0 + br %r14 +sie_fault: + lg %r14,__LC_THREAD_INFO # pointer thread_info struct + ni __TI_flags+6(%r14),255-(_TIF_SIE>>8) + lg %r14,__SF_EMPTY+8(%r15) # load guest register save area + stmg %r0,%r13,0(%r14) # save guest gprs 0-13 + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + lghi %r2,-EFAULT + br %r14 + + .align 8 +.Lsie_loop: + .quad sie_loop +.Lsie_done: + .quad sie_done + + .section __ex_table,"a" + .quad sie_loop,sie_fault + .previous +#endif + .section .rodata, "a" #define SYSCALL(esa,esame,emu) .long esame .globl sys_call_table diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S index fb317bf2c378..2d781bab37bb 100644 --- a/arch/s390/kernel/head.S +++ b/arch/s390/kernel/head.S @@ -22,6 +22,7 @@ */ #include <linux/init.h> +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/page.h> @@ -383,8 +384,7 @@ iplstart: # doesn't need a builtin ipl record. # .org 0x800 - .globl start -start: +ENTRY(start) stm %r0,%r15,0x07b0 # store registers basr %r12,%r0 .base: @@ -448,8 +448,7 @@ start: # or linload or SALIPL # .org 0x10000 - .globl startup -startup: +ENTRY(startup) basr %r13,0 # get base .LPG0: xc 0x200(256),0x200 # partially clear lowcore diff --git a/arch/s390/kernel/head31.S b/arch/s390/kernel/head31.S index b8f8dc126102..f21954b44dc1 100644 --- a/arch/s390/kernel/head31.S +++ b/arch/s390/kernel/head31.S @@ -11,13 +11,13 @@ */ #include <linux/init.h> +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/page.h> __HEAD - .globl startup_continue -startup_continue: +ENTRY(startup_continue) basr %r13,0 # get base .LPG1: @@ -45,7 +45,7 @@ startup_continue: # virtual and never return ... .align 8 .Lentry:.long 0x00080000,0x80000000 + _stext -.Lctl: .long 0x04b50002 # cr0: various things +.Lctl: .long 0x04b50000 # cr0: various things .long 0 # cr1: primary space segment table .long .Lduct # cr2: dispatchable unit control table .long 0 # cr3: instruction authorization @@ -78,8 +78,7 @@ startup_continue: .Lbase_cc: .long sched_clock_base_cc - .globl _ehead -_ehead: +ENTRY(_ehead) #ifdef CONFIG_SHARED_KERNEL .org 0x100000 - 0x11000 # head.o ends at 0x11000 @@ -88,8 +87,8 @@ _ehead: # # startup-code, running in absolute addressing mode # - .globl _stext -_stext: basr %r13,0 # get base +ENTRY(_stext) + basr %r13,0 # get base .LPG3: # check control registers stctl %c0,%c15,0(%r15) diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index cdef68717416..ae5d492b069e 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -11,13 +11,13 @@ */ #include <linux/init.h> +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/page.h> __HEAD - .globl startup_continue -startup_continue: +ENTRY(startup_continue) larl %r1,sched_clock_base_cc mvc 0(8,%r1),__LC_LAST_UPDATE_CLOCK larl %r13,.LPG1 # get base @@ -46,7 +46,7 @@ startup_continue: .align 16 .LPG1: .Lentry:.quad 0x0000000180000000,_stext -.Lctl: .quad 0x04350002 # cr0: various things +.Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space .quad 0 # cr1: primary space segment table .quad .Lduct # cr2: dispatchable unit control table .quad 0 # cr3: instruction authorization @@ -76,8 +76,7 @@ startup_continue: .long 0x80000000,0,0,0 # invalid access-list entries .endr - .globl _ehead -_ehead: +ENTRY(_ehead) #ifdef CONFIG_SHARED_KERNEL .org 0x100000 - 0x11000 # head.o ends at 0x11000 @@ -86,8 +85,8 @@ _ehead: # # startup-code, running in absolute addressing mode # - .globl _stext -_stext: basr %r13,0 # get base +ENTRY(_stext) + basr %r13,0 # get base .LPG3: # check control registers stctg %c0,%c15,0(%r15) diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index a689070be287..04361d5a4279 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -45,11 +45,13 @@ * - halt * - power off * - reipl + * - restart */ #define ON_PANIC_STR "on_panic" #define ON_HALT_STR "on_halt" #define ON_POFF_STR "on_poff" #define ON_REIPL_STR "on_reboot" +#define ON_RESTART_STR "on_restart" struct shutdown_action; struct shutdown_trigger { @@ -1544,17 +1546,20 @@ static char vmcmd_on_reboot[128]; static char vmcmd_on_panic[128]; static char vmcmd_on_halt[128]; static char vmcmd_on_poff[128]; +static char vmcmd_on_restart[128]; DEFINE_IPL_ATTR_STR_RW(vmcmd, on_reboot, "%s\n", "%s\n", vmcmd_on_reboot); DEFINE_IPL_ATTR_STR_RW(vmcmd, on_panic, "%s\n", "%s\n", vmcmd_on_panic); DEFINE_IPL_ATTR_STR_RW(vmcmd, on_halt, "%s\n", "%s\n", vmcmd_on_halt); DEFINE_IPL_ATTR_STR_RW(vmcmd, on_poff, "%s\n", "%s\n", vmcmd_on_poff); +DEFINE_IPL_ATTR_STR_RW(vmcmd, on_restart, "%s\n", "%s\n", vmcmd_on_restart); static struct attribute *vmcmd_attrs[] = { &sys_vmcmd_on_reboot_attr.attr, &sys_vmcmd_on_panic_attr.attr, &sys_vmcmd_on_halt_attr.attr, &sys_vmcmd_on_poff_attr.attr, + &sys_vmcmd_on_restart_attr.attr, NULL, }; @@ -1576,6 +1581,8 @@ static void vmcmd_run(struct shutdown_trigger *trigger) cmd = vmcmd_on_halt; else if (strcmp(trigger->name, ON_POFF_STR) == 0) cmd = vmcmd_on_poff; + else if (strcmp(trigger->name, ON_RESTART_STR) == 0) + cmd = vmcmd_on_restart; else return; @@ -1707,6 +1714,34 @@ static void do_panic(void) stop_run(&on_panic_trigger); } +/* on restart */ + +static struct shutdown_trigger on_restart_trigger = {ON_RESTART_STR, + &reipl_action}; + +static ssize_t on_restart_show(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return sprintf(page, "%s\n", on_restart_trigger.action->name); +} + +static ssize_t on_restart_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t len) +{ + return set_trigger(buf, &on_restart_trigger, len); +} + +static struct kobj_attribute on_restart_attr = + __ATTR(on_restart, 0644, on_restart_show, on_restart_store); + +void do_restart(void) +{ + smp_send_stop(); + on_restart_trigger.action->fn(&on_restart_trigger); + stop_run(&on_restart_trigger); +} + /* on halt */ static struct shutdown_trigger on_halt_trigger = {ON_HALT_STR, &stop_action}; @@ -1783,7 +1818,9 @@ static void __init shutdown_triggers_init(void) if (sysfs_create_file(&shutdown_actions_kset->kobj, &on_poff_attr.attr)) goto fail; - + if (sysfs_create_file(&shutdown_actions_kset->kobj, + &on_restart_attr.attr)) + goto fail; return; fail: panic("shutdown_triggers_init failed\n"); @@ -1959,6 +1996,12 @@ static void do_reset_calls(void) { struct reset_call *reset; +#ifdef CONFIG_64BIT + if (diag308_set_works) { + diag308_reset(); + return; + } +#endif list_for_each_entry(reset, &rcall, list) reset->fn(); } diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index ea5099c9709c..1f4050d45f78 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -1,19 +1,28 @@ /* - * Copyright IBM Corp. 2004,2010 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), - * Thomas Spatzier (tspat@de.ibm.com) + * Copyright IBM Corp. 2004,2011 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, + * Holger Smolinski <Holger.Smolinski@de.ibm.com>, + * Thomas Spatzier <tspat@de.ibm.com>, * * This file contains interrupt related functions. */ -#include <linux/module.h> -#include <linux/kernel.h> #include <linux/kernel_stat.h> #include <linux/interrupt.h> #include <linux/seq_file.h> -#include <linux/cpu.h> #include <linux/proc_fs.h> #include <linux/profile.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/ftrace.h> +#include <linux/errno.h> +#include <linux/slab.h> +#include <linux/cpu.h> +#include <asm/irq_regs.h> +#include <asm/cputime.h> +#include <asm/lowcore.h> +#include <asm/irq.h> +#include "entry.h" struct irq_class { char *name; @@ -32,6 +41,7 @@ static const struct irq_class intrclass_names[] = { {.name = "VRT", .desc = "[EXT] Virtio" }, {.name = "SCP", .desc = "[EXT] Service Call" }, {.name = "IUC", .desc = "[EXT] IUCV" }, + {.name = "CPM", .desc = "[EXT] CPU Measurement" }, {.name = "QAI", .desc = "[I/O] QDIO Adapter Interrupt" }, {.name = "QDI", .desc = "[I/O] QDIO Interrupt" }, {.name = "DAS", .desc = "[I/O] DASD" }, @@ -78,16 +88,6 @@ int show_interrupts(struct seq_file *p, void *v) } /* - * For compatibilty only. S/390 specific setup of interrupts et al. is done - * much later in init_channel_subsystem(). - */ -void __init -init_IRQ(void) -{ - /* nothing... */ -} - -/* * Switch to the asynchronous interrupt stack for softirq execution. */ asmlinkage void do_softirq(void) @@ -133,3 +133,134 @@ void init_irq_proc(void) create_prof_cpu_mask(root_irq_dir); } #endif + +/* + * ext_int_hash[index] is the list head for all external interrupts that hash + * to this index. + */ +static struct list_head ext_int_hash[256]; + +struct ext_int_info { + ext_int_handler_t handler; + u16 code; + struct list_head entry; + struct rcu_head rcu; +}; + +/* ext_int_hash_lock protects the handler lists for external interrupts */ +DEFINE_SPINLOCK(ext_int_hash_lock); + +static void __init init_external_interrupts(void) +{ + int idx; + + for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++) + INIT_LIST_HEAD(&ext_int_hash[idx]); +} + +static inline int ext_hash(u16 code) +{ + return (code + (code >> 9)) & 0xff; +} + +static void ext_int_hash_update(struct rcu_head *head) +{ + struct ext_int_info *p = container_of(head, struct ext_int_info, rcu); + + kfree(p); +} + +int register_external_interrupt(u16 code, ext_int_handler_t handler) +{ + struct ext_int_info *p; + unsigned long flags; + int index; + + p = kmalloc(sizeof(*p), GFP_ATOMIC); + if (!p) + return -ENOMEM; + p->code = code; + p->handler = handler; + index = ext_hash(code); + + spin_lock_irqsave(&ext_int_hash_lock, flags); + list_add_rcu(&p->entry, &ext_int_hash[index]); + spin_unlock_irqrestore(&ext_int_hash_lock, flags); + return 0; +} +EXPORT_SYMBOL(register_external_interrupt); + +int unregister_external_interrupt(u16 code, ext_int_handler_t handler) +{ + struct ext_int_info *p; + unsigned long flags; + int index = ext_hash(code); + + spin_lock_irqsave(&ext_int_hash_lock, flags); + list_for_each_entry_rcu(p, &ext_int_hash[index], entry) + if (p->code == code && p->handler == handler) { + list_del_rcu(&p->entry); + call_rcu(&p->rcu, ext_int_hash_update); + } + spin_unlock_irqrestore(&ext_int_hash_lock, flags); + return 0; +} +EXPORT_SYMBOL(unregister_external_interrupt); + +void __irq_entry do_extint(struct pt_regs *regs, unsigned int ext_int_code, + unsigned int param32, unsigned long param64) +{ + struct pt_regs *old_regs; + unsigned short code; + struct ext_int_info *p; + int index; + + code = (unsigned short) ext_int_code; + old_regs = set_irq_regs(regs); + s390_idle_check(regs, S390_lowcore.int_clock, + S390_lowcore.async_enter_timer); + irq_enter(); + if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) + /* Serve timer interrupts first. */ + clock_comparator_work(); + kstat_cpu(smp_processor_id()).irqs[EXTERNAL_INTERRUPT]++; + if (code != 0x1004) + __get_cpu_var(s390_idle).nohz_delay = 1; + + index = ext_hash(code); + rcu_read_lock(); + list_for_each_entry_rcu(p, &ext_int_hash[index], entry) + if (likely(p->code == code)) + p->handler(ext_int_code, param32, param64); + rcu_read_unlock(); + irq_exit(); + set_irq_regs(old_regs); +} + +void __init init_IRQ(void) +{ + init_external_interrupts(); +} + +static DEFINE_SPINLOCK(sc_irq_lock); +static int sc_irq_refcount; + +void service_subclass_irq_register(void) +{ + spin_lock(&sc_irq_lock); + if (!sc_irq_refcount) + ctl_set_bit(0, 9); + sc_irq_refcount++; + spin_unlock(&sc_irq_lock); +} +EXPORT_SYMBOL(service_subclass_irq_register); + +void service_subclass_irq_unregister(void) +{ + spin_lock(&sc_irq_lock); + sc_irq_refcount--; + if (!sc_irq_refcount) + ctl_clear_bit(0, 9); + spin_unlock(&sc_irq_lock); +} +EXPORT_SYMBOL(service_subclass_irq_unregister); diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c new file mode 100644 index 000000000000..44cc06bedf77 --- /dev/null +++ b/arch/s390/kernel/jump_label.c @@ -0,0 +1,59 @@ +/* + * Jump label s390 support + * + * Copyright IBM Corp. 2011 + * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> + */ +#include <linux/module.h> +#include <linux/uaccess.h> +#include <linux/stop_machine.h> +#include <linux/jump_label.h> +#include <asm/ipl.h> + +#ifdef HAVE_JUMP_LABEL + +struct insn { + u16 opcode; + s32 offset; +} __packed; + +struct insn_args { + unsigned long *target; + struct insn *insn; + ssize_t size; +}; + +static int __arch_jump_label_transform(void *data) +{ + struct insn_args *args = data; + int rc; + + rc = probe_kernel_write(args->target, args->insn, args->size); + WARN_ON_ONCE(rc < 0); + return 0; +} + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + struct insn_args args; + struct insn insn; + + if (type == JUMP_LABEL_ENABLE) { + /* brcl 15,offset */ + insn.opcode = 0xc0f4; + insn.offset = (entry->target - entry->code) >> 1; + } else { + /* brcl 0,0 */ + insn.opcode = 0xc004; + insn.offset = 0; + } + + args.target = (void *) entry->code; + args.insn = &insn; + args.size = JUMP_LABEL_NOP_SIZE; + + stop_machine(__arch_jump_label_transform, &args, NULL); +} + +#endif diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 1e6a55795628..7e2c38ba1373 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -5,21 +5,19 @@ * */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> .section .kprobes.text, "ax" - .globl ftrace_stub -ftrace_stub: +ENTRY(ftrace_stub) br %r14 - .globl _mcount -_mcount: +ENTRY(_mcount) #ifdef CONFIG_DYNAMIC_FTRACE br %r14 - .globl ftrace_caller -ftrace_caller: +ENTRY(ftrace_caller) #endif stm %r2,%r5,16(%r15) bras %r1,2f @@ -41,8 +39,7 @@ ftrace_caller: #ifdef CONFIG_FUNCTION_GRAPH_TRACER l %r2,100(%r15) l %r3,152(%r15) - .globl ftrace_graph_caller -ftrace_graph_caller: +ENTRY(ftrace_graph_caller) # The bras instruction gets runtime patched to call prepare_ftrace_return. # See ftrace_enable_ftrace_graph_caller. The patched instruction is: # bras %r14,prepare_ftrace_return @@ -56,8 +53,7 @@ ftrace_graph_caller: #ifdef CONFIG_FUNCTION_GRAPH_TRACER - .globl return_to_handler -return_to_handler: +ENTRY(return_to_handler) stm %r2,%r5,16(%r15) st %r14,56(%r15) lr %r0,%r15 diff --git a/arch/s390/kernel/mcount64.S b/arch/s390/kernel/mcount64.S index e73667286ac0..f70cadec68fc 100644 --- a/arch/s390/kernel/mcount64.S +++ b/arch/s390/kernel/mcount64.S @@ -5,21 +5,19 @@ * */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> .section .kprobes.text, "ax" - .globl ftrace_stub -ftrace_stub: +ENTRY(ftrace_stub) br %r14 - .globl _mcount -_mcount: +ENTRY(_mcount) #ifdef CONFIG_DYNAMIC_FTRACE br %r14 - .globl ftrace_caller -ftrace_caller: +ENTRY(ftrace_caller) #endif larl %r1,function_trace_stop icm %r1,0xf,0(%r1) @@ -37,8 +35,7 @@ ftrace_caller: #ifdef CONFIG_FUNCTION_GRAPH_TRACER lg %r2,168(%r15) lg %r3,272(%r15) - .globl ftrace_graph_caller -ftrace_graph_caller: +ENTRY(ftrace_graph_caller) # The bras instruction gets runtime patched to call prepare_ftrace_return. # See ftrace_enable_ftrace_graph_caller. The patched instruction is: # bras %r14,prepare_ftrace_return @@ -52,8 +49,7 @@ ftrace_graph_caller: #ifdef CONFIG_FUNCTION_GRAPH_TRACER - .globl return_to_handler -return_to_handler: +ENTRY(return_to_handler) stmg %r2,%r5,32(%r15) lgr %r1,%r15 aghi %r15,-160 diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index f7167ee4604c..dfcb3436bad0 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -45,13 +45,6 @@ #define PLT_ENTRY_SIZE 20 #endif /* CONFIG_64BIT */ -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - /* Free memory returned from module_alloc */ void module_free(struct module *mod, void *module_region) { @@ -176,15 +169,6 @@ module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, return 0; } -int -apply_relocate(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, - unsigned int relsec, struct module *me) -{ - printk(KERN_ERR "module %s: RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, struct module *me) @@ -409,7 +393,3 @@ int module_finalize(const Elf_Ehdr *hdr, me->arch.syminfo = NULL; return 0; } - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index a895e69379f7..541a7509faeb 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -9,41 +9,26 @@ #include <linux/compiler.h> #include <linux/cpu.h> -#include <linux/errno.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/mm.h> -#include <linux/fs.h> #include <linux/smp.h> -#include <linux/stddef.h> #include <linux/slab.h> -#include <linux/unistd.h> -#include <linux/ptrace.h> -#include <linux/vmalloc.h> -#include <linux/user.h> #include <linux/interrupt.h> -#include <linux/delay.h> -#include <linux/reboot.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/notifier.h> #include <linux/tick.h> -#include <linux/elfcore.h> -#include <linux/kernel_stat.h> #include <linux/personality.h> #include <linux/syscalls.h> #include <linux/compat.h> #include <linux/kprobes.h> #include <linux/random.h> -#include <asm/compat.h> -#include <asm/uaccess.h> -#include <asm/pgtable.h> +#include <linux/module.h> #include <asm/system.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/irq.h> #include <asm/timer.h> #include <asm/nmi.h> +#include <asm/compat.h> #include <asm/smp.h> #include "entry.h" diff --git a/arch/s390/kernel/reipl.S b/arch/s390/kernel/reipl.S index cb899d9f8505..303d961c3bb5 100644 --- a/arch/s390/kernel/reipl.S +++ b/arch/s390/kernel/reipl.S @@ -6,14 +6,15 @@ * Author(s): Holger Smolinski (Holger.Smolinski@de.ibm.com) */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> # # do_reipl_asm # Parameter: r2 = schid of reipl device # - .globl do_reipl_asm -do_reipl_asm: basr %r13,0 +ENTRY(do_reipl_asm) + basr %r13,0 .Lpg0: lpsw .Lnewpsw-.Lpg0(%r13) .Lpg1: # do store status of all registers diff --git a/arch/s390/kernel/reipl64.S b/arch/s390/kernel/reipl64.S index 9eabbc90795d..e690975403f4 100644 --- a/arch/s390/kernel/reipl64.S +++ b/arch/s390/kernel/reipl64.S @@ -1,35 +1,79 @@ /* - * Copyright IBM Corp 2000,2009 + * Copyright IBM Corp 2000,2011 * Author(s): Holger Smolinski <Holger.Smolinski@de.ibm.com>, * Denis Joseph Barrow, */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> # +# store_status +# +# Prerequisites to run this function: +# - Prefix register is set to zero +# - Original prefix register is stored in "dump_prefix_page" +# - Lowcore protection is off +# +ENTRY(store_status) + /* Save register one and load save area base */ + stg %r1,__LC_SAVE_AREA_64(%r0) + lghi %r1,SAVE_AREA_BASE + /* General purpose registers */ + stmg %r0,%r15,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + lg %r2,__LC_SAVE_AREA_64(%r0) + stg %r2,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE+8(%r1) + /* Control registers */ + stctg %c0,%c15,__LC_CREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Access registers */ + stam %a0,%a15,__LC_AREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Floating point registers */ + std %f0, 0x00 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f1, 0x08 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f2, 0x10 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f3, 0x18 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f4, 0x20 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f5, 0x28 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f6, 0x30 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f7, 0x38 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f8, 0x40 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f9, 0x48 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f10,0x50 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f11,0x58 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f12,0x60 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f13,0x68 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f14,0x70 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f15,0x78 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Floating point control register */ + stfpc __LC_FP_CREG_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* CPU timer */ + stpt __LC_CPU_TIMER_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Saved prefix register */ + larl %r2,dump_prefix_page + mvc __LC_PREFIX_SAVE_AREA-SAVE_AREA_BASE(4,%r1),0(%r2) + /* Clock comparator - seven bytes */ + larl %r2,.Lclkcmp + stckc 0(%r2) + mvc __LC_CLOCK_COMP_SAVE_AREA-SAVE_AREA_BASE + 1(7,%r1),1(%r2) + /* Program status word */ + epsw %r2,%r3 + st %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 0(%r1) + st %r3,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 4(%r1) + larl %r2,store_status + stg %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 8(%r1) + br %r14 +.align 8 +.Lclkcmp: .quad 0x0000000000000000 + +# # do_reipl_asm # Parameter: r2 = schid of reipl device # - .globl do_reipl_asm -do_reipl_asm: basr %r13,0 +ENTRY(do_reipl_asm) + basr %r13,0 .Lpg0: lpswe .Lnewpsw-.Lpg0(%r13) -.Lpg1: # do store status of all registers - - stg %r1,.Lregsave-.Lpg0(%r13) - lghi %r1,0x1000 - stmg %r0,%r15,__LC_GPREGS_SAVE_AREA-0x1000(%r1) - lg %r0,.Lregsave-.Lpg0(%r13) - stg %r0,__LC_GPREGS_SAVE_AREA-0x1000+8(%r1) - stctg %c0,%c15,__LC_CREGS_SAVE_AREA-0x1000(%r1) - stam %a0,%a15,__LC_AREGS_SAVE_AREA-0x1000(%r1) - lg %r10,.Ldump_pfx-.Lpg0(%r13) - mvc __LC_PREFIX_SAVE_AREA-0x1000(4,%r1),0(%r10) - stfpc __LC_FP_CREG_SAVE_AREA-0x1000(%r1) - stckc .Lclkcmp-.Lpg0(%r13) - mvc __LC_CLOCK_COMP_SAVE_AREA-0x1000(7,%r1),.Lclkcmp-.Lpg0(%r13) - stpt __LC_CPU_TIMER_SAVE_AREA-0x1000(%r1) - stg %r13, __LC_PSW_SAVE_AREA-0x1000+8(%r1) +.Lpg1: brasl %r14,store_status lctlg %c6,%c6,.Lall-.Lpg0(%r13) lgr %r1,%r2 @@ -66,10 +110,7 @@ do_reipl_asm: basr %r13,0 st %r14,.Ldispsw+12-.Lpg0(%r13) lpswe .Ldispsw-.Lpg0(%r13) .align 8 -.Lclkcmp: .quad 0x0000000000000000 .Lall: .quad 0x00000000ff000000 -.Ldump_pfx: .quad dump_prefix_page -.Lregsave: .quad 0x0000000000000000 .align 16 /* * These addresses have to be 31 bit otherwise diff --git a/arch/s390/kernel/relocate_kernel.S b/arch/s390/kernel/relocate_kernel.S index 3b456b80bcee..c91d70aede91 100644 --- a/arch/s390/kernel/relocate_kernel.S +++ b/arch/s390/kernel/relocate_kernel.S @@ -8,6 +8,8 @@ * */ +#include <linux/linkage.h> + /* * moves the new kernel to its destination... * %r2 = pointer to first kimage_entry_t @@ -22,8 +24,7 @@ */ .text - .globl relocate_kernel - relocate_kernel: +ENTRY(relocate_kernel) basr %r13,0 # base address .base: stnsm sys_msk-.base(%r13),0xfb # disable DAT @@ -112,6 +113,7 @@ .byte 0 .align 8 relocate_kernel_end: + .align 8 .globl relocate_kernel_len relocate_kernel_len: .quad relocate_kernel_end - relocate_kernel diff --git a/arch/s390/kernel/relocate_kernel64.S b/arch/s390/kernel/relocate_kernel64.S index 1f9ea2067b59..7c3ce589a7f0 100644 --- a/arch/s390/kernel/relocate_kernel64.S +++ b/arch/s390/kernel/relocate_kernel64.S @@ -8,6 +8,8 @@ * */ +#include <linux/linkage.h> + /* * moves the new kernel to its destination... * %r2 = pointer to first kimage_entry_t @@ -23,8 +25,7 @@ */ .text - .globl relocate_kernel - relocate_kernel: +ENTRY(relocate_kernel) basr %r13,0 # base address .base: stnsm sys_msk-.base(%r13),0xfb # disable DAT @@ -115,6 +116,7 @@ .byte 0 .align 8 relocate_kernel_end: + .align 8 .globl relocate_kernel_len relocate_kernel_len: .quad relocate_kernel_end - relocate_kernel diff --git a/arch/s390/kernel/s390_ext.c b/arch/s390/kernel/s390_ext.c deleted file mode 100644 index 185029919c4d..000000000000 --- a/arch/s390/kernel/s390_ext.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright IBM Corp. 1999,2010 - * Author(s): Holger Smolinski <Holger.Smolinski@de.ibm.com>, - * Martin Schwidefsky <schwidefsky@de.ibm.com>, - */ - -#include <linux/kernel_stat.h> -#include <linux/interrupt.h> -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/ftrace.h> -#include <linux/errno.h> -#include <linux/slab.h> -#include <asm/s390_ext.h> -#include <asm/irq_regs.h> -#include <asm/cputime.h> -#include <asm/lowcore.h> -#include <asm/irq.h> -#include "entry.h" - -struct ext_int_info { - struct ext_int_info *next; - ext_int_handler_t handler; - __u16 code; -}; - -/* - * ext_int_hash[index] is the start of the list for all external interrupts - * that hash to this index. With the current set of external interrupts - * (0x1202 external call, 0x1004 cpu timer, 0x2401 hwc console, 0x4000 - * iucv and 0x2603 pfault) this is always the first element. - */ -static struct ext_int_info *ext_int_hash[256]; - -static inline int ext_hash(__u16 code) -{ - return (code + (code >> 9)) & 0xff; -} - -int register_external_interrupt(__u16 code, ext_int_handler_t handler) -{ - struct ext_int_info *p; - int index; - - p = kmalloc(sizeof(*p), GFP_ATOMIC); - if (!p) - return -ENOMEM; - p->code = code; - p->handler = handler; - index = ext_hash(code); - p->next = ext_int_hash[index]; - ext_int_hash[index] = p; - return 0; -} -EXPORT_SYMBOL(register_external_interrupt); - -int unregister_external_interrupt(__u16 code, ext_int_handler_t handler) -{ - struct ext_int_info *p, *q; - int index; - - index = ext_hash(code); - q = NULL; - p = ext_int_hash[index]; - while (p) { - if (p->code == code && p->handler == handler) - break; - q = p; - p = p->next; - } - if (!p) - return -ENOENT; - if (q) - q->next = p->next; - else - ext_int_hash[index] = p->next; - kfree(p); - return 0; -} -EXPORT_SYMBOL(unregister_external_interrupt); - -void __irq_entry do_extint(struct pt_regs *regs, unsigned int ext_int_code, - unsigned int param32, unsigned long param64) -{ - struct pt_regs *old_regs; - unsigned short code; - struct ext_int_info *p; - int index; - - code = (unsigned short) ext_int_code; - old_regs = set_irq_regs(regs); - s390_idle_check(regs, S390_lowcore.int_clock, - S390_lowcore.async_enter_timer); - irq_enter(); - if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) - /* Serve timer interrupts first. */ - clock_comparator_work(); - kstat_cpu(smp_processor_id()).irqs[EXTERNAL_INTERRUPT]++; - if (code != 0x1004) - __get_cpu_var(s390_idle).nohz_delay = 1; - index = ext_hash(code); - for (p = ext_int_hash[index]; p; p = p->next) { - if (likely(p->code == code)) - p->handler(ext_int_code, param32, param64); - } - irq_exit(); - set_irq_regs(old_regs); -} diff --git a/arch/s390/kernel/s390_ksyms.c b/arch/s390/kernel/s390_ksyms.c index 656fcbb9bd83..57b536649b00 100644 --- a/arch/s390/kernel/s390_ksyms.c +++ b/arch/s390/kernel/s390_ksyms.c @@ -1,6 +1,10 @@ #include <linux/module.h> +#include <linux/kvm_host.h> #include <asm/ftrace.h> #ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL(_mcount); #endif +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +EXPORT_SYMBOL(sie64a); +#endif diff --git a/arch/s390/kernel/sclp.S b/arch/s390/kernel/sclp.S index 2e82fdd89320..95792d846bb6 100644 --- a/arch/s390/kernel/sclp.S +++ b/arch/s390/kernel/sclp.S @@ -8,6 +8,8 @@ * */ +#include <linux/linkage.h> + LC_EXT_NEW_PSW = 0x58 # addr of ext int handler LC_EXT_NEW_PSW_64 = 0x1b0 # addr of ext int handler 64 bit LC_EXT_INT_PARAM = 0x80 # addr of ext int parameter @@ -260,8 +262,7 @@ _sclp_print: # R2 = 0 on success, 1 on failure # - .globl _sclp_print_early -_sclp_print_early: +ENTRY(_sclp_print_early) stm %r6,%r15,24(%r15) # save registers ahi %r15,-96 # create stack frame #ifdef CONFIG_64BIT diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index f5434d1ecb31..7b371c37061d 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -305,8 +305,7 @@ static int set_amode_and_uaccess(unsigned long user_amode, */ static int __init early_parse_switch_amode(char *p) { - if (user_mode != SECONDARY_SPACE_MODE) - user_mode = PRIMARY_SPACE_MODE; + user_mode = PRIMARY_SPACE_MODE; return 0; } early_param("switch_amode", early_parse_switch_amode); @@ -315,10 +314,6 @@ static int __init early_parse_user_mode(char *p) { if (p && strcmp(p, "primary") == 0) user_mode = PRIMARY_SPACE_MODE; -#ifdef CONFIG_S390_EXEC_PROTECT - else if (p && strcmp(p, "secondary") == 0) - user_mode = SECONDARY_SPACE_MODE; -#endif else if (!p || strcmp(p, "home") == 0) user_mode = HOME_SPACE_MODE; else @@ -327,31 +322,9 @@ static int __init early_parse_user_mode(char *p) } early_param("user_mode", early_parse_user_mode); -#ifdef CONFIG_S390_EXEC_PROTECT -/* - * Enable execute protection? - */ -static int __init early_parse_noexec(char *p) -{ - if (!strncmp(p, "off", 3)) - return 0; - user_mode = SECONDARY_SPACE_MODE; - return 0; -} -early_param("noexec", early_parse_noexec); -#endif /* CONFIG_S390_EXEC_PROTECT */ - static void setup_addressing_mode(void) { - if (user_mode == SECONDARY_SPACE_MODE) { - if (set_amode_and_uaccess(PSW_ASC_SECONDARY, - PSW32_ASC_SECONDARY)) - pr_info("Execute protection active, " - "mvcos available\n"); - else - pr_info("Execute protection active, " - "mvcos not available\n"); - } else if (user_mode == PRIMARY_SPACE_MODE) { + if (user_mode == PRIMARY_SPACE_MODE) { if (set_amode_and_uaccess(PSW_ASC_PRIMARY, PSW32_ASC_PRIMARY)) pr_info("Address spaces switched, " "mvcos available\n"); @@ -373,7 +346,7 @@ setup_lowcore(void) lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0); lc->restart_psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY; lc->restart_psw.addr = - PSW_ADDR_AMODE | (unsigned long) restart_int_handler; + PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler; if (user_mode != HOME_SPACE_MODE) lc->restart_psw.mask |= PSW_ASC_HOME; lc->external_new_psw.mask = psw_kernel_bits; @@ -556,6 +529,27 @@ static void __init setup_memory_end(void) memory_end = memory_size; } +void *restart_stack __attribute__((__section__(".data"))); + +/* + * Setup new PSW and allocate stack for PSW restart interrupt + */ +static void __init setup_restart_psw(void) +{ + psw_t psw; + + restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0); + restart_stack += ASYNC_SIZE; + + /* + * Setup restart PSW for absolute zero lowcore. This is necesary + * if PSW restart is done on an offline CPU that has lowcore zero + */ + psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY; + psw.addr = PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler; + copy_to_absolute_zero(&S390_lowcore.restart_psw, &psw, sizeof(psw)); +} + static void __init setup_memory(void) { @@ -758,6 +752,7 @@ static void __init setup_hwcaps(void) strcpy(elf_platform, "z10"); break; case 0x2817: + case 0x2818: strcpy(elf_platform, "z196"); break; } @@ -819,6 +814,7 @@ setup_arch(char **cmdline_p) setup_addressing_mode(); setup_memory(); setup_resources(); + setup_restart_psw(); setup_lowcore(); cpu_init(); diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index abbb3c3c7aab..9a40e1cc5ec3 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -57,17 +57,15 @@ typedef struct */ SYSCALL_DEFINE3(sigsuspend, int, history0, int, history1, old_sigset_t, mask) { - mask &= _BLOCKABLE; - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - siginitset(¤t->blocked, mask); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + sigset_t blocked; + current->saved_sigmask = current->blocked; + mask &= _BLOCKABLE; + siginitset(&blocked, mask); + set_current_blocked(&blocked); set_current_state(TASK_INTERRUPTIBLE); schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - + set_restore_sigmask(); return -ERESTARTNOHAND; } @@ -172,18 +170,11 @@ SYSCALL_DEFINE0(sigreturn) goto badframe; if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE)) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - + set_current_blocked(&set); if (restore_sigregs(regs, &frame->sregs)) goto badframe; - return regs->gprs[2]; - badframe: force_sig(SIGSEGV, current); return 0; @@ -199,21 +190,14 @@ SYSCALL_DEFINE0(rt_sigreturn) goto badframe; if (__copy_from_user(&set.sig, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - + set_current_blocked(&set); if (restore_sigregs(regs, &frame->uc.uc_mcontext)) goto badframe; - if (do_sigaltstack(&frame->uc.uc_stack, NULL, regs->gprs[15]) == -EFAULT) goto badframe; return regs->gprs[2]; - badframe: force_sig(SIGSEGV, current); return 0; @@ -385,14 +369,11 @@ give_sigsegv: return -EFAULT; } -/* - * OK, we're invoking a handler - */ - -static int -handle_signal(unsigned long sig, struct k_sigaction *ka, - siginfo_t *info, sigset_t *oldset, struct pt_regs * regs) +static int handle_signal(unsigned long sig, struct k_sigaction *ka, + siginfo_t *info, sigset_t *oldset, + struct pt_regs *regs) { + sigset_t blocked; int ret; /* Set up the stack frame */ @@ -400,17 +381,13 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, ret = setup_rt_frame(sig, ka, info, oldset, regs); else ret = setup_frame(sig, ka, oldset, regs); - - if (ret == 0) { - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NODEFER)) - sigaddset(¤t->blocked,sig); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - } - - return ret; + if (ret) + return ret; + sigorsets(&blocked, ¤t->blocked, &ka->sa.sa_mask); + if (!(ka->sa.sa_flags & SA_NODEFER)) + sigaddset(&blocked, sig); + set_current_blocked(&blocked); + return 0; } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 63a97db83f96..6ab16ac64d29 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -44,7 +44,6 @@ #include <asm/sigp.h> #include <asm/pgalloc.h> #include <asm/irq.h> -#include <asm/s390_ext.h> #include <asm/cpcmd.h> #include <asm/tlbflush.h> #include <asm/timer.h> @@ -165,12 +164,12 @@ static void do_ext_call_interrupt(unsigned int ext_int_code, kstat_cpu(smp_processor_id()).irqs[EXTINT_IPI]++; /* * handle bit signal external calls - * - * For the ec_schedule signal we have to do nothing. All the work - * is done automatically when we return from the interrupt. */ bits = xchg(&S390_lowcore.ext_call_fast, 0); + if (test_bit(ec_schedule, &bits)) + scheduler_ipi(); + if (test_bit(ec_call_function, &bits)) generic_smp_call_function_interrupt(); @@ -263,7 +262,7 @@ void smp_ctl_set_bit(int cr, int bit) memset(&parms.orvals, 0, sizeof(parms.orvals)); memset(&parms.andvals, 0xff, sizeof(parms.andvals)); - parms.orvals[cr] = 1 << bit; + parms.orvals[cr] = 1UL << bit; on_each_cpu(smp_ctl_bit_callback, &parms, 1); } EXPORT_SYMBOL(smp_ctl_set_bit); @@ -277,7 +276,7 @@ void smp_ctl_clear_bit(int cr, int bit) memset(&parms.orvals, 0, sizeof(parms.orvals)); memset(&parms.andvals, 0xff, sizeof(parms.andvals)); - parms.andvals[cr] = ~(1L << bit); + parms.andvals[cr] = ~(1UL << bit); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } EXPORT_SYMBOL(smp_ctl_clear_bit); @@ -335,7 +334,7 @@ static int smp_rescan_cpus_sigp(cpumask_t avail) smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN; if (!cpu_stopped(logical_cpu)) continue; - cpu_set(logical_cpu, cpu_present_map); + set_cpu_present(logical_cpu, true); smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED; logical_cpu = cpumask_next(logical_cpu, &avail); if (logical_cpu >= nr_cpu_ids) @@ -367,7 +366,7 @@ static int smp_rescan_cpus_sclp(cpumask_t avail) continue; __cpu_logical_map[logical_cpu] = cpu_id; smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN; - cpu_set(logical_cpu, cpu_present_map); + set_cpu_present(logical_cpu, true); if (cpu >= info->configured) smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY; else @@ -385,7 +384,7 @@ static int __smp_rescan_cpus(void) { cpumask_t avail; - cpus_xor(avail, cpu_possible_map, cpu_present_map); + cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask); if (smp_use_sigp_detection) return smp_rescan_cpus_sigp(avail); else @@ -453,23 +452,27 @@ out: */ int __cpuinit start_secondary(void *cpuvoid) { - /* Setup the cpu */ cpu_init(); preempt_disable(); - /* Enable TOD clock interrupts on the secondary cpu. */ init_cpu_timer(); - /* Enable cpu timer interrupts on the secondary cpu. */ init_cpu_vtimer(); - /* Enable pfault pseudo page faults on this cpu. */ pfault_init(); - /* call cpu notifiers */ notify_cpu_starting(smp_processor_id()); - /* Mark this cpu as online */ ipi_call_lock(); - cpu_set(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), true); ipi_call_unlock(); - /* Switch on interrupts */ + __ctl_clear_bit(0, 28); /* Disable lowcore protection */ + S390_lowcore.restart_psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY; + S390_lowcore.restart_psw.addr = + PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler; + __ctl_set_bit(0, 28); /* Enable lowcore protection */ + /* + * Wait until the cpu which brought this one up marked it + * active before enabling interrupts. + */ + while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask)) + cpu_relax(); local_irq_enable(); /* cpu_idle will call schedule for us */ cpu_idle(); @@ -508,7 +511,11 @@ static int __cpuinit smp_alloc_lowcore(int cpu) memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512); lowcore->async_stack = async_stack + ASYNC_SIZE; lowcore->panic_stack = panic_stack + PAGE_SIZE; - + lowcore->restart_psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY; + lowcore->restart_psw.addr = + PSW_ADDR_AMODE | (unsigned long) restart_int_handler; + if (user_mode != HOME_SPACE_MODE) + lowcore->restart_psw.mask |= PSW_ASC_HOME; #ifndef CONFIG_64BIT if (MACHINE_HAS_IEEE) { unsigned long save_area; @@ -644,7 +651,7 @@ int __cpu_disable(void) struct ec_creg_mask_parms cr_parms; int cpu = smp_processor_id(); - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); /* Disable pfault pseudo page faults on this cpu. */ pfault_fini(); @@ -654,8 +661,9 @@ int __cpu_disable(void) /* disable all external interrupts */ cr_parms.orvals[0] = 0; - cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 | - 1 << 11 | 1 << 10 | 1 << 6 | 1 << 4); + cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 11 | + 1 << 10 | 1 << 9 | 1 << 6 | 1 << 5 | + 1 << 4); /* disable all I/O interrupts */ cr_parms.orvals[6] = 0; cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 | @@ -681,7 +689,7 @@ void __cpu_die(unsigned int cpu) atomic_dec(&init_mm.context.attach_count); } -void cpu_die(void) +void __noreturn cpu_die(void) { idle_task_exit(); while (sigp(smp_processor_id(), sigp_stop) == sigp_busy) @@ -738,8 +746,8 @@ void __init smp_prepare_boot_cpu(void) BUG_ON(smp_processor_id() != 0); current_thread_info()->cpu = 0; - cpu_set(0, cpu_present_map); - cpu_set(0, cpu_online_map); + set_cpu_present(0, true); + set_cpu_online(0, true); S390_lowcore.percpu_offset = __per_cpu_offset[0]; current_set[0] = current; smp_cpu_state[0] = CPU_STATE_CONFIGURED; @@ -1016,21 +1024,21 @@ int __ref smp_rescan_cpus(void) get_online_cpus(); mutex_lock(&smp_cpu_state_mutex); - newcpus = cpu_present_map; + cpumask_copy(&newcpus, cpu_present_mask); rc = __smp_rescan_cpus(); if (rc) goto out; - cpus_andnot(newcpus, cpu_present_map, newcpus); - for_each_cpu_mask(cpu, newcpus) { + cpumask_andnot(&newcpus, cpu_present_mask, &newcpus); + for_each_cpu(cpu, &newcpus) { rc = smp_add_present_cpu(cpu); if (rc) - cpu_clear(cpu, cpu_present_map); + set_cpu_present(cpu, false); } rc = 0; out: mutex_unlock(&smp_cpu_state_mutex); put_online_cpus(); - if (!cpus_empty(newcpus)) + if (!cpumask_empty(&newcpus)) topology_schedule_update(); return rc; } diff --git a/arch/s390/kernel/switch_cpu.S b/arch/s390/kernel/switch_cpu.S index 20530dd2eab1..bfe070bc7659 100644 --- a/arch/s390/kernel/switch_cpu.S +++ b/arch/s390/kernel/switch_cpu.S @@ -5,6 +5,7 @@ * */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> @@ -16,9 +17,7 @@ # %r6 - destination cpu .section .text - .align 4 - .globl smp_switch_to_cpu -smp_switch_to_cpu: +ENTRY(smp_switch_to_cpu) stm %r6,%r15,__SF_GPRS(%r15) lr %r1,%r15 ahi %r15,-STACK_FRAME_OVERHEAD @@ -33,8 +32,7 @@ smp_switch_to_cpu: brc 2,2b /* busy, try again */ 3: j 3b - .globl smp_restart_cpu -smp_restart_cpu: +ENTRY(smp_restart_cpu) basr %r13,0 0: la %r1,.gprregs_addr-0b(%r13) l %r1,0(%r1) diff --git a/arch/s390/kernel/switch_cpu64.S b/arch/s390/kernel/switch_cpu64.S index 5be3f43898f9..fcc42d799e41 100644 --- a/arch/s390/kernel/switch_cpu64.S +++ b/arch/s390/kernel/switch_cpu64.S @@ -5,6 +5,7 @@ * */ +#include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> @@ -16,9 +17,7 @@ # %r6 - destination cpu .section .text - .align 4 - .globl smp_switch_to_cpu -smp_switch_to_cpu: +ENTRY(smp_switch_to_cpu) stmg %r6,%r15,__SF_GPRS(%r15) lgr %r1,%r15 aghi %r15,-STACK_FRAME_OVERHEAD @@ -31,8 +30,7 @@ smp_switch_to_cpu: brc 2,2b /* busy, try again */ 3: j 3b - .globl smp_restart_cpu -smp_restart_cpu: +ENTRY(smp_restart_cpu) larl %r1,.gprregs lmg %r0,%r15,0(%r1) 1: sigp %r0,%r5,__SIGP_SENSE /* Wait for calling CPU */ diff --git a/arch/s390/kernel/swsusp_asm64.S b/arch/s390/kernel/swsusp_asm64.S index 1f066e46e83e..51bcdb50a230 100644 --- a/arch/s390/kernel/swsusp_asm64.S +++ b/arch/s390/kernel/swsusp_asm64.S @@ -7,6 +7,7 @@ * Michael Holzheu <holzheu@linux.vnet.ibm.com> */ +#include <linux/linkage.h> #include <asm/page.h> #include <asm/ptrace.h> #include <asm/thread_info.h> @@ -22,9 +23,7 @@ * This function runs with disabled interrupts. */ .section .text - .align 4 - .globl swsusp_arch_suspend -swsusp_arch_suspend: +ENTRY(swsusp_arch_suspend) stmg %r6,%r15,__SF_GPRS(%r15) lgr %r1,%r15 aghi %r15,-STACK_FRAME_OVERHEAD @@ -112,8 +111,7 @@ swsusp_arch_suspend: * Then we return to the function that called swsusp_arch_suspend(). * swsusp_arch_resume() runs with disabled interrupts. */ - .globl swsusp_arch_resume -swsusp_arch_resume: +ENTRY(swsusp_arch_resume) stmg %r6,%r15,__SF_GPRS(%r15) lgr %r1,%r15 aghi %r15,-STACK_FRAME_OVERHEAD diff --git a/arch/s390/kernel/syscalls.S b/arch/s390/kernel/syscalls.S index 9c65fd4ddce0..6ee39ef8fe4a 100644 --- a/arch/s390/kernel/syscalls.S +++ b/arch/s390/kernel/syscalls.S @@ -347,3 +347,4 @@ SYSCALL(sys_name_to_handle_at,sys_name_to_handle_at,sys_name_to_handle_at_wrappe SYSCALL(sys_open_by_handle_at,sys_open_by_handle_at,compat_sys_open_by_handle_at_wrapper) SYSCALL(sys_clock_adjtime,sys_clock_adjtime,compat_sys_clock_adjtime_wrapper) SYSCALL(sys_syncfs,sys_syncfs,sys_syncfs_wrapper) +SYSCALL(sys_setns,sys_setns,sys_setns_wrapper) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 87be655557aa..dff933065ab6 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -41,7 +41,6 @@ #include <linux/kprobes.h> #include <asm/uaccess.h> #include <asm/delay.h> -#include <asm/s390_ext.h> #include <asm/div64.h> #include <asm/vdso.h> #include <asm/irq.h> @@ -810,7 +809,7 @@ static int etr_sync_clock_stop(struct etr_aib *aib, int port) etr_sync.etr_port = port; get_online_cpus(); atomic_set(&etr_sync.cpus, num_online_cpus() - 1); - rc = stop_machine(etr_sync_clock, &etr_sync, &cpu_online_map); + rc = stop_machine(etr_sync_clock, &etr_sync, cpu_online_mask); put_online_cpus(); return rc; } @@ -1579,7 +1578,7 @@ static void stp_work_fn(struct work_struct *work) memset(&stp_sync, 0, sizeof(stp_sync)); get_online_cpus(); atomic_set(&stp_sync.cpus, num_online_cpus() - 1); - stop_machine(stp_sync_clock, &stp_sync, &cpu_online_map); + stop_machine(stp_sync_clock, &stp_sync, cpu_online_mask); put_online_cpus(); if (!check_sync_clock()) diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 94b06c31fc8a..0cd340b72632 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -17,7 +17,6 @@ #include <linux/smp.h> #include <linux/cpuset.h> #include <asm/delay.h> -#include <asm/s390_ext.h> #define PTF_HORIZONTAL (0UL) #define PTF_VERTICAL (1UL) @@ -52,20 +51,20 @@ static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) { cpumask_t mask; - cpus_clear(mask); + cpumask_clear(&mask); if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) { cpumask_copy(&mask, cpumask_of(cpu)); return mask; } while (info) { - if (cpu_isset(cpu, info->mask)) { + if (cpumask_test_cpu(cpu, &info->mask)) { mask = info->mask; break; } info = info->next; } - if (cpus_empty(mask)) - mask = cpumask_of_cpu(cpu); + if (cpumask_empty(&mask)) + cpumask_copy(&mask, cpumask_of(cpu)); return mask; } @@ -85,10 +84,10 @@ static void add_cpus_to_mask(struct topology_cpu *tl_cpu, if (cpu_logical_map(lcpu) != rcpu) continue; #ifdef CONFIG_SCHED_BOOK - cpu_set(lcpu, book->mask); + cpumask_set_cpu(lcpu, &book->mask); cpu_book_id[lcpu] = book->id; #endif - cpu_set(lcpu, core->mask); + cpumask_set_cpu(lcpu, &core->mask); cpu_core_id[lcpu] = core->id; smp_cpu_polarization[lcpu] = tl_cpu->pp; } @@ -101,13 +100,13 @@ static void clear_masks(void) info = &core_info; while (info) { - cpus_clear(info->mask); + cpumask_clear(&info->mask); info = info->next; } #ifdef CONFIG_SCHED_BOOK info = &book_info; while (info) { - cpus_clear(info->mask); + cpumask_clear(&info->mask); info = info->next; } #endif diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index b5a4a739b477..ffabcd9d3363 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -18,7 +18,7 @@ #include <linux/kernel.h> #include <linux/string.h> #include <linux/errno.h> -#include <linux/tracehook.h> +#include <linux/ptrace.h> #include <linux/timer.h> #include <linux/mm.h> #include <linux/smp.h> @@ -36,22 +36,17 @@ #include <asm/system.h> #include <asm/uaccess.h> #include <asm/io.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/mathemu.h> #include <asm/cpcmd.h> -#include <asm/s390_ext.h> #include <asm/lowcore.h> #include <asm/debug.h> #include "entry.h" -pgm_check_handler_t *pgm_check_table[128]; +void (*pgm_check_table[128])(struct pt_regs *, long, unsigned long); int show_unhandled_signals; -extern pgm_check_handler_t do_protection_exception; -extern pgm_check_handler_t do_dat_exception; -extern pgm_check_handler_t do_asce_exception; - #define stack_pointer ({ void **sp; asm("la %0,0(15)" : "=&d" (sp)); sp; }) #ifndef CONFIG_64BIT @@ -330,10 +325,17 @@ static inline void __user *get_psw_address(struct pt_regs *regs, void __kprobes do_per_trap(struct pt_regs *regs) { + siginfo_t info; + if (notify_die(DIE_SSTEP, "sstep", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; - if (tracehook_consider_fatal_signal(current, SIGTRAP)) - force_sig(SIGTRAP, current); + if (!current->ptrace) + return; + info.si_signo = SIGTRAP; + info.si_errno = 0; + info.si_code = TRAP_HWBKPT; + info.si_addr = (void *) current->thread.per_event.address; + force_sig_info(SIGTRAP, &info, current); } static void default_trap_handler(struct pt_regs *regs, long pgm_int_code, @@ -426,9 +428,13 @@ static void __kprobes illegal_op(struct pt_regs *regs, long pgm_int_code, if (get_user(*((__u16 *) opcode), (__u16 __user *) location)) return; if (*((__u16 *) opcode) == S390_BREAKPOINT_U16) { - if (tracehook_consider_fatal_signal(current, SIGTRAP)) - force_sig(SIGTRAP, current); - else + if (current->ptrace) { + info.si_signo = SIGTRAP; + info.si_errno = 0; + info.si_code = TRAP_BRKPT; + info.si_addr = location; + force_sig_info(SIGTRAP, &info, current); + } else signal = SIGILL; #ifdef CONFIG_MATHEMU } else if (opcode[0] == 0xb3) { @@ -490,9 +496,8 @@ static void __kprobes illegal_op(struct pt_regs *regs, long pgm_int_code, #ifdef CONFIG_MATHEMU -asmlinkage void specification_exception(struct pt_regs *regs, - long pgm_int_code, - unsigned long trans_exc_code) +void specification_exception(struct pt_regs *regs, long pgm_int_code, + unsigned long trans_exc_code) { __u8 opcode[6]; __u16 __user *location = NULL; @@ -649,7 +654,7 @@ static void space_switch_exception(struct pt_regs *regs, long pgm_int_code, do_trap(pgm_int_code, SIGILL, "space switch event", regs, &info); } -asmlinkage void __kprobes kernel_stack_overflow(struct pt_regs * regs) +void __kprobes kernel_stack_overflow(struct pt_regs * regs) { bust_spinlocks(1); printk("Kernel stack overflow.\n"); diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index d13e8755a8cc..8ad2b34ad151 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -22,6 +22,9 @@ obj-y += vdso32_wrapper.o extra-y += vdso32.lds CPPFLAGS_vdso32.lds += -P -C -U$(ARCH) +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n + # Force dependency (incbin is bad) $(obj)/vdso32_wrapper.o : $(obj)/vdso32.so diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 449352dda9cd..2a8ddfd12a5b 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -22,6 +22,9 @@ obj-y += vdso64_wrapper.o extra-y += vdso64.lds CPPFLAGS_vdso64.lds += -P -C -U$(ARCH) +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n + # Force dependency (incbin is bad) $(obj)/vdso64_wrapper.o : $(obj)/vdso64.so diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 1bc18cdb525b..56fe6bc81fee 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -77,7 +77,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); INIT_DATA_SECTION(0x100) - PERCPU(0x100, PAGE_SIZE) + PERCPU_SECTION(0x100) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 5e8ead4b4aba..2d6228f60cd6 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -22,10 +22,10 @@ #include <linux/cpu.h> #include <linux/kprobes.h> -#include <asm/s390_ext.h> #include <asm/timer.h> #include <asm/irq_regs.h> #include <asm/cputime.h> +#include <asm/irq.h> static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index f66a1bdbb61d..a21634173a66 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -37,6 +37,5 @@ config KVM # OK, it's a little counter-intuitive to do this, but it puts it neatly under # the virtualization menu. source drivers/vhost/Kconfig -source drivers/virtio/Kconfig endif # VIRTUALIZATION diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 860d26514c08..3975722bb19d 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -10,5 +10,5 @@ common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o) ccflags-y := -Ivirt/kvm -Iarch/s390/kvm -kvm-objs := $(common-objs) kvm-s390.o sie64a.o intercept.o interrupt.o priv.o sigp.o diag.o +kvm-objs := $(common-objs) kvm-s390.o intercept.o interrupt.o priv.o sigp.o diag.o obj-$(CONFIG_KVM) += kvm.o diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h index 03c716a0f01f..c86f6ae43f76 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/gaccess.h @@ -1,5 +1,5 @@ /* - * gaccess.h - access guest memory + * access.h - access guest memory * * Copyright IBM Corp. 2008,2009 * @@ -22,20 +22,13 @@ static inline void __user *__guestaddr_to_user(struct kvm_vcpu *vcpu, unsigned long guestaddr) { unsigned long prefix = vcpu->arch.sie_block->prefix; - unsigned long origin = vcpu->arch.sie_block->gmsor; - unsigned long memsize = kvm_s390_vcpu_get_memsize(vcpu); if (guestaddr < 2 * PAGE_SIZE) guestaddr += prefix; else if ((guestaddr >= prefix) && (guestaddr < prefix + 2 * PAGE_SIZE)) guestaddr -= prefix; - if (guestaddr > memsize) - return (void __user __force *) ERR_PTR(-EFAULT); - - guestaddr += origin; - - return (void __user *) guestaddr; + return (void __user *) gmap_fault(guestaddr, vcpu->arch.gmap); } static inline int get_guest_u64(struct kvm_vcpu *vcpu, unsigned long guestaddr, @@ -141,11 +134,11 @@ static inline int put_guest_u8(struct kvm_vcpu *vcpu, unsigned long guestaddr, static inline int __copy_to_guest_slow(struct kvm_vcpu *vcpu, unsigned long guestdest, - const void *from, unsigned long n) + void *from, unsigned long n) { int rc; unsigned long i; - const u8 *data = from; + u8 *data = from; for (i = 0; i < n; i++) { rc = put_guest_u8(vcpu, guestdest++, *(data++)); @@ -155,12 +148,95 @@ static inline int __copy_to_guest_slow(struct kvm_vcpu *vcpu, return 0; } +static inline int __copy_to_guest_fast(struct kvm_vcpu *vcpu, + unsigned long guestdest, + void *from, unsigned long n) +{ + int r; + void __user *uptr; + unsigned long size; + + if (guestdest + n < guestdest) + return -EFAULT; + + /* simple case: all within one segment table entry? */ + if ((guestdest & PMD_MASK) == ((guestdest+n) & PMD_MASK)) { + uptr = (void __user *) gmap_fault(guestdest, vcpu->arch.gmap); + + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); + + r = copy_to_user(uptr, from, n); + + if (r) + r = -EFAULT; + + goto out; + } + + /* copy first segment */ + uptr = (void __user *)gmap_fault(guestdest, vcpu->arch.gmap); + + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); + + size = PMD_SIZE - (guestdest & ~PMD_MASK); + + r = copy_to_user(uptr, from, size); + + if (r) { + r = -EFAULT; + goto out; + } + from += size; + n -= size; + guestdest += size; + + /* copy full segments */ + while (n >= PMD_SIZE) { + uptr = (void __user *)gmap_fault(guestdest, vcpu->arch.gmap); + + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); + + r = copy_to_user(uptr, from, PMD_SIZE); + + if (r) { + r = -EFAULT; + goto out; + } + from += PMD_SIZE; + n -= PMD_SIZE; + guestdest += PMD_SIZE; + } + + /* copy the tail segment */ + if (n) { + uptr = (void __user *)gmap_fault(guestdest, vcpu->arch.gmap); + + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); + + r = copy_to_user(uptr, from, n); + + if (r) + r = -EFAULT; + } +out: + return r; +} + +static inline int copy_to_guest_absolute(struct kvm_vcpu *vcpu, + unsigned long guestdest, + void *from, unsigned long n) +{ + return __copy_to_guest_fast(vcpu, guestdest, from, n); +} + static inline int copy_to_guest(struct kvm_vcpu *vcpu, unsigned long guestdest, - const void *from, unsigned long n) + void *from, unsigned long n) { unsigned long prefix = vcpu->arch.sie_block->prefix; - unsigned long origin = vcpu->arch.sie_block->gmsor; - unsigned long memsize = kvm_s390_vcpu_get_memsize(vcpu); if ((guestdest < 2 * PAGE_SIZE) && (guestdest + n > 2 * PAGE_SIZE)) goto slowpath; @@ -177,15 +253,7 @@ static inline int copy_to_guest(struct kvm_vcpu *vcpu, unsigned long guestdest, else if ((guestdest >= prefix) && (guestdest < prefix + 2 * PAGE_SIZE)) guestdest -= prefix; - if (guestdest + n > memsize) - return -EFAULT; - - if (guestdest + n < guestdest) - return -EFAULT; - - guestdest += origin; - - return copy_to_user((void __user *) guestdest, from, n); + return __copy_to_guest_fast(vcpu, guestdest, from, n); slowpath: return __copy_to_guest_slow(vcpu, guestdest, from, n); } @@ -206,74 +274,113 @@ static inline int __copy_from_guest_slow(struct kvm_vcpu *vcpu, void *to, return 0; } -static inline int copy_from_guest(struct kvm_vcpu *vcpu, void *to, - unsigned long guestsrc, unsigned long n) +static inline int __copy_from_guest_fast(struct kvm_vcpu *vcpu, void *to, + unsigned long guestsrc, + unsigned long n) { - unsigned long prefix = vcpu->arch.sie_block->prefix; - unsigned long origin = vcpu->arch.sie_block->gmsor; - unsigned long memsize = kvm_s390_vcpu_get_memsize(vcpu); + int r; + void __user *uptr; + unsigned long size; - if ((guestsrc < 2 * PAGE_SIZE) && (guestsrc + n > 2 * PAGE_SIZE)) - goto slowpath; + if (guestsrc + n < guestsrc) + return -EFAULT; - if ((guestsrc < prefix) && (guestsrc + n > prefix)) - goto slowpath; + /* simple case: all within one segment table entry? */ + if ((guestsrc & PMD_MASK) == ((guestsrc+n) & PMD_MASK)) { + uptr = (void __user *) gmap_fault(guestsrc, vcpu->arch.gmap); - if ((guestsrc < prefix + 2 * PAGE_SIZE) - && (guestsrc + n > prefix + 2 * PAGE_SIZE)) - goto slowpath; + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); - if (guestsrc < 2 * PAGE_SIZE) - guestsrc += prefix; - else if ((guestsrc >= prefix) && (guestsrc < prefix + 2 * PAGE_SIZE)) - guestsrc -= prefix; + r = copy_from_user(to, uptr, n); - if (guestsrc + n > memsize) - return -EFAULT; + if (r) + r = -EFAULT; - if (guestsrc + n < guestsrc) - return -EFAULT; + goto out; + } - guestsrc += origin; + /* copy first segment */ + uptr = (void __user *)gmap_fault(guestsrc, vcpu->arch.gmap); - return copy_from_user(to, (void __user *) guestsrc, n); -slowpath: - return __copy_from_guest_slow(vcpu, to, guestsrc, n); -} + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); -static inline int copy_to_guest_absolute(struct kvm_vcpu *vcpu, - unsigned long guestdest, - const void *from, unsigned long n) -{ - unsigned long origin = vcpu->arch.sie_block->gmsor; - unsigned long memsize = kvm_s390_vcpu_get_memsize(vcpu); + size = PMD_SIZE - (guestsrc & ~PMD_MASK); - if (guestdest + n > memsize) - return -EFAULT; + r = copy_from_user(to, uptr, size); - if (guestdest + n < guestdest) - return -EFAULT; + if (r) { + r = -EFAULT; + goto out; + } + to += size; + n -= size; + guestsrc += size; + + /* copy full segments */ + while (n >= PMD_SIZE) { + uptr = (void __user *)gmap_fault(guestsrc, vcpu->arch.gmap); + + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); + + r = copy_from_user(to, uptr, PMD_SIZE); + + if (r) { + r = -EFAULT; + goto out; + } + to += PMD_SIZE; + n -= PMD_SIZE; + guestsrc += PMD_SIZE; + } + + /* copy the tail segment */ + if (n) { + uptr = (void __user *)gmap_fault(guestsrc, vcpu->arch.gmap); - guestdest += origin; + if (IS_ERR((void __force *) uptr)) + return PTR_ERR((void __force *) uptr); - return copy_to_user((void __user *) guestdest, from, n); + r = copy_from_user(to, uptr, n); + + if (r) + r = -EFAULT; + } +out: + return r; } static inline int copy_from_guest_absolute(struct kvm_vcpu *vcpu, void *to, unsigned long guestsrc, unsigned long n) { - unsigned long origin = vcpu->arch.sie_block->gmsor; - unsigned long memsize = kvm_s390_vcpu_get_memsize(vcpu); + return __copy_from_guest_fast(vcpu, to, guestsrc, n); +} - if (guestsrc + n > memsize) - return -EFAULT; +static inline int copy_from_guest(struct kvm_vcpu *vcpu, void *to, + unsigned long guestsrc, unsigned long n) +{ + unsigned long prefix = vcpu->arch.sie_block->prefix; - if (guestsrc + n < guestsrc) - return -EFAULT; + if ((guestsrc < 2 * PAGE_SIZE) && (guestsrc + n > 2 * PAGE_SIZE)) + goto slowpath; - guestsrc += origin; + if ((guestsrc < prefix) && (guestsrc + n > prefix)) + goto slowpath; + + if ((guestsrc < prefix + 2 * PAGE_SIZE) + && (guestsrc + n > prefix + 2 * PAGE_SIZE)) + goto slowpath; + + if (guestsrc < 2 * PAGE_SIZE) + guestsrc += prefix; + else if ((guestsrc >= prefix) && (guestsrc < prefix + 2 * PAGE_SIZE)) + guestsrc -= prefix; - return copy_from_user(to, (void __user *) guestsrc, n); + return __copy_from_guest_fast(vcpu, to, guestsrc, n); +slowpath: + return __copy_from_guest_slow(vcpu, to, guestsrc, n); } #endif diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index f7b6df45d8be..c7c51898984e 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -105,6 +105,7 @@ static intercept_handler_t instruction_handlers[256] = { [0xae] = kvm_s390_handle_sigp, [0xb2] = kvm_s390_handle_b2, [0xb7] = handle_lctl, + [0xe5] = kvm_s390_handle_e5, [0xeb] = handle_lctlg, }; @@ -159,22 +160,42 @@ static int handle_stop(struct kvm_vcpu *vcpu) static int handle_validity(struct kvm_vcpu *vcpu) { + unsigned long vmaddr; int viwhy = vcpu->arch.sie_block->ipb >> 16; int rc; vcpu->stat.exit_validity++; - if ((viwhy == 0x37) && (vcpu->arch.sie_block->prefix - <= kvm_s390_vcpu_get_memsize(vcpu) - 2*PAGE_SIZE)) { - rc = fault_in_pages_writeable((char __user *) - vcpu->arch.sie_block->gmsor + - vcpu->arch.sie_block->prefix, - 2*PAGE_SIZE); - if (rc) + if (viwhy == 0x37) { + vmaddr = gmap_fault(vcpu->arch.sie_block->prefix, + vcpu->arch.gmap); + if (IS_ERR_VALUE(vmaddr)) { + rc = -EOPNOTSUPP; + goto out; + } + rc = fault_in_pages_writeable((char __user *) vmaddr, + PAGE_SIZE); + if (rc) { + /* user will receive sigsegv, exit to user */ + rc = -EOPNOTSUPP; + goto out; + } + vmaddr = gmap_fault(vcpu->arch.sie_block->prefix + PAGE_SIZE, + vcpu->arch.gmap); + if (IS_ERR_VALUE(vmaddr)) { + rc = -EOPNOTSUPP; + goto out; + } + rc = fault_in_pages_writeable((char __user *) vmaddr, + PAGE_SIZE); + if (rc) { /* user will receive sigsegv, exit to user */ rc = -EOPNOTSUPP; + goto out; + } } else rc = -EOPNOTSUPP; +out: if (rc) VCPU_EVENT(vcpu, 2, "unhandled validity intercept code %d", viwhy); diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 35c21bf910c5..c9aeb4b4d0b8 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -128,6 +128,10 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, if (rc == -EFAULT) exception = 1; + rc = put_guest_u16(vcpu, __LC_CPU_ADDRESS, inti->emerg.code); + if (rc == -EFAULT) + exception = 1; + rc = copy_to_guest(vcpu, __LC_EXT_OLD_PSW, &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); if (rc == -EFAULT) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 30ca85cce314..f17296e4fc89 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -62,6 +62,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "instruction_chsc", VCPU_STAT(instruction_chsc) }, { "instruction_stsi", VCPU_STAT(instruction_stsi) }, { "instruction_stfl", VCPU_STAT(instruction_stfl) }, + { "instruction_tprot", VCPU_STAT(instruction_tprot) }, { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, @@ -189,7 +190,13 @@ int kvm_arch_init_vm(struct kvm *kvm) debug_register_view(kvm->arch.dbf, &debug_sprintf_view); VM_EVENT(kvm, 3, "%s", "vm created"); + kvm->arch.gmap = gmap_alloc(current->mm); + if (!kvm->arch.gmap) + goto out_nogmap; + return 0; +out_nogmap: + debug_unregister(kvm->arch.dbf); out_nodbf: free_page((unsigned long)(kvm->arch.sca)); out_err: @@ -234,11 +241,13 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_free_vcpus(kvm); free_page((unsigned long)(kvm->arch.sca)); debug_unregister(kvm->arch.dbf); + gmap_free(kvm->arch.gmap); } /* Section: vcpu related */ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) { + vcpu->arch.gmap = vcpu->kvm->arch.gmap; return 0; } @@ -284,8 +293,7 @@ static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) { - atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH); - set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests); + atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | CPUSTAT_SM); vcpu->arch.sie_block->ecb = 6; vcpu->arch.sie_block->eca = 0xC1002001U; vcpu->arch.sie_block->fac = (int) (long) facilities; @@ -453,6 +461,7 @@ static void __vcpu_run(struct kvm_vcpu *vcpu) local_irq_disable(); kvm_guest_enter(); local_irq_enable(); + gmap_enable(vcpu->arch.gmap); VCPU_EVENT(vcpu, 6, "entering sie flags %x", atomic_read(&vcpu->arch.sie_block->cpuflags)); if (sie64a(vcpu->arch.sie_block, vcpu->arch.guest_gprs)) { @@ -461,6 +470,7 @@ static void __vcpu_run(struct kvm_vcpu *vcpu) } VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", vcpu->arch.sie_block->icptcode); + gmap_disable(vcpu->arch.gmap); local_irq_disable(); kvm_guest_exit(); local_irq_enable(); @@ -474,17 +484,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) sigset_t sigsaved; rerun_vcpu: - if (vcpu->requests) - if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) - kvm_s390_vcpu_set_mem(vcpu); - - /* verify, that memory has been registered */ - if (!vcpu->arch.sie_block->gmslm) { - vcpu_put(vcpu); - VCPU_EVENT(vcpu, 3, "%s", "no memory registered to run vcpu"); - return -EINVAL; - } - if (vcpu->sigset_active) sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); @@ -545,7 +544,7 @@ rerun_vcpu: return rc; } -static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, const void *from, +static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from, unsigned long n, int prefix) { if (prefix) @@ -562,7 +561,7 @@ static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, const void *from, */ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) { - const unsigned char archmode = 1; + unsigned char archmode = 1; int prefix; if (addr == KVM_S390_STORE_STATUS_NOADDR) { @@ -680,10 +679,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (mem->guest_phys_addr) return -EINVAL; - if (mem->userspace_addr & (PAGE_SIZE - 1)) + if (mem->userspace_addr & 0xffffful) return -EINVAL; - if (mem->memory_size & (PAGE_SIZE - 1)) + if (mem->memory_size & 0xffffful) return -EINVAL; if (!user_alloc) @@ -697,15 +696,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot old, int user_alloc) { - int i; - struct kvm_vcpu *vcpu; + int rc; - /* request update of sie control block for all available vcpus */ - kvm_for_each_vcpu(i, vcpu, kvm) { - if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) - continue; - kvm_s390_inject_sigp_stop(vcpu, ACTION_RELOADVCPU_ON_STOP); - } + + rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, + mem->guest_phys_addr, mem->memory_size); + if (rc) + printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); + return; } void kvm_arch_flush_shadow(struct kvm *kvm) @@ -731,6 +729,7 @@ static int __init kvm_s390_init(void) } memcpy(facilities, S390_lowcore.stfle_fac_list, 16); facilities[0] &= 0xff00fff3f47c0000ULL; + facilities[1] &= 0x201c000000000000ULL; return 0; } diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index a7b7586626db..99b0b7597115 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -58,35 +58,9 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code); int kvm_s390_inject_sigp_stop(struct kvm_vcpu *vcpu, int action); -static inline long kvm_s390_vcpu_get_memsize(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.sie_block->gmslm - - vcpu->arch.sie_block->gmsor - - VIRTIODESCSPACE + 1ul; -} - -static inline void kvm_s390_vcpu_set_mem(struct kvm_vcpu *vcpu) -{ - int idx; - struct kvm_memory_slot *mem; - struct kvm_memslots *memslots; - - idx = srcu_read_lock(&vcpu->kvm->srcu); - memslots = kvm_memslots(vcpu->kvm); - - mem = &memslots->memslots[0]; - - vcpu->arch.sie_block->gmsor = mem->userspace_addr; - vcpu->arch.sie_block->gmslm = - mem->userspace_addr + - (mem->npages << PAGE_SHIFT) + - VIRTIODESCSPACE - 1ul; - - srcu_read_unlock(&vcpu->kvm->srcu, idx); -} - /* implemented in priv.c */ int kvm_s390_handle_b2(struct kvm_vcpu *vcpu); +int kvm_s390_handle_e5(struct kvm_vcpu *vcpu); /* implemented in sigp.c */ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index 73c47bd95db3..391626361084 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -326,3 +326,52 @@ int kvm_s390_handle_b2(struct kvm_vcpu *vcpu) } return -EOPNOTSUPP; } + +static int handle_tprot(struct kvm_vcpu *vcpu) +{ + int base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; + int disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16; + int base2 = (vcpu->arch.sie_block->ipb & 0xf000) >> 12; + int disp2 = vcpu->arch.sie_block->ipb & 0x0fff; + u64 address1 = disp1 + base1 ? vcpu->arch.guest_gprs[base1] : 0; + u64 address2 = disp2 + base2 ? vcpu->arch.guest_gprs[base2] : 0; + struct vm_area_struct *vma; + + vcpu->stat.instruction_tprot++; + + /* we only handle the Linux memory detection case: + * access key == 0 + * guest DAT == off + * everything else goes to userspace. */ + if (address2 & 0xf0) + return -EOPNOTSUPP; + if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) + return -EOPNOTSUPP; + + + down_read(¤t->mm->mmap_sem); + vma = find_vma(current->mm, + (unsigned long) __guestaddr_to_user(vcpu, address1)); + if (!vma) { + up_read(¤t->mm->mmap_sem); + return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); + } + + vcpu->arch.sie_block->gpsw.mask &= ~(3ul << 44); + if (!(vma->vm_flags & VM_WRITE) && (vma->vm_flags & VM_READ)) + vcpu->arch.sie_block->gpsw.mask |= (1ul << 44); + if (!(vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_READ)) + vcpu->arch.sie_block->gpsw.mask |= (2ul << 44); + + up_read(¤t->mm->mmap_sem); + return 0; +} + +int kvm_s390_handle_e5(struct kvm_vcpu *vcpu) +{ + /* For e5xx... instructions we only handle TPROT */ + if ((vcpu->arch.sie_block->ipa & 0x00ff) == 0x01) + return handle_tprot(vcpu); + return -EOPNOTSUPP; +} + diff --git a/arch/s390/kvm/sie64a.S b/arch/s390/kvm/sie64a.S deleted file mode 100644 index ab0e041ac54c..000000000000 --- a/arch/s390/kvm/sie64a.S +++ /dev/null @@ -1,96 +0,0 @@ -/* - * sie64a.S - low level sie call - * - * Copyright IBM Corp. 2008,2010 - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License (version 2 only) - * as published by the Free Software Foundation. - * - * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> - * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com> - */ - -#include <linux/errno.h> -#include <asm/asm-offsets.h> -#include <asm/setup.h> -#include <asm/asm-offsets.h> -#include <asm/ptrace.h> -#include <asm/thread_info.h> - -_TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING) - -/* - * offsets into stackframe - * SP_ = offsets into stack sie64 is called with - * SPI_ = offsets into irq stack - */ -SP_GREGS = __SF_EMPTY -SP_HOOK = __SF_EMPTY+8 -SP_GPP = __SF_EMPTY+16 -SPI_PSW = STACK_FRAME_OVERHEAD + __PT_PSW - - - .macro SPP newpp - tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_SPP - jz 0f - .insn s,0xb2800000,\newpp -0: - .endm - -sie_irq_handler: - SPP __LC_CMF_HPP # set host id - larl %r2,sie_inst - clg %r2,SPI_PSW+8(0,%r15) # intercepted sie - jne 1f - xc __LC_SIE_HOOK(8),__LC_SIE_HOOK - lg %r2,__LC_THREAD_INFO # pointer thread_info struct - tm __TI_flags+7(%r2),_TIF_EXIT_SIE - jz 0f - larl %r2,sie_exit # work pending, leave sie - stg %r2,SPI_PSW+8(0,%r15) - br %r14 -0: larl %r2,sie_reenter # re-enter with guest id - stg %r2,SPI_PSW+8(0,%r15) -1: br %r14 - -/* - * sie64a calling convention: - * %r2 pointer to sie control block - * %r3 guest register save area - */ - .globl sie64a -sie64a: - stg %r3,SP_GREGS(%r15) # save guest register save area - stmg %r6,%r14,__SF_GPRS(%r15) # save registers on entry - lgr %r14,%r2 # pointer to sie control block - larl %r5,sie_irq_handler - stg %r2,SP_GPP(%r15) - stg %r5,SP_HOOK(%r15) # save hook target - lmg %r0,%r13,0(%r3) # load guest gprs 0-13 -sie_reenter: - mvc __LC_SIE_HOOK(8),SP_HOOK(%r15) - SPP SP_GPP(%r15) # set guest id -sie_inst: - sie 0(%r14) - xc __LC_SIE_HOOK(8),__LC_SIE_HOOK - SPP __LC_CMF_HPP # set host id -sie_exit: - lg %r14,SP_GREGS(%r15) - stmg %r0,%r13,0(%r14) # save guest gprs 0-13 - lghi %r2,0 - lmg %r6,%r14,__SF_GPRS(%r15) - br %r14 - -sie_err: - xc __LC_SIE_HOOK(8),__LC_SIE_HOOK - SPP __LC_CMF_HPP # set host id - lg %r14,SP_GREGS(%r15) - stmg %r0,%r13,0(%r14) # save guest gprs 0-13 - lghi %r2,-EFAULT - lmg %r6,%r14,__SF_GPRS(%r15) - br %r14 - - .section __ex_table,"a" - .quad sie_inst,sie_err - .previous diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c index 702276f5e2fa..d6a50c1fb2e6 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/sigp.c @@ -189,10 +189,8 @@ static int __sigp_set_prefix(struct kvm_vcpu *vcpu, u16 cpu_addr, u32 address, /* make sure that the new value is valid memory */ address = address & 0x7fffe000u; - if ((copy_from_user(&tmp, (void __user *) - (address + vcpu->arch.sie_block->gmsor) , 1)) || - (copy_from_user(&tmp, (void __user *)(address + - vcpu->arch.sie_block->gmsor + PAGE_SIZE), 1))) { + if (copy_from_guest_absolute(vcpu, &tmp, address, 1) || + copy_from_guest_absolute(vcpu, &tmp, address + PAGE_SIZE, 1)) { *reg |= SIGP_STAT_INVALID_PARAMETER; return 1; /* invalid parameter */ } diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index 0f53110e1d09..a65229d91c92 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -12,6 +12,7 @@ #include <linux/module.h> #include <linux/irqflags.h> #include <linux/interrupt.h> +#include <asm/div64.h> void __delay(unsigned long loops) { @@ -116,3 +117,17 @@ void udelay_simple(unsigned long long usecs) while (get_clock() < end) cpu_relax(); } + +void __ndelay(unsigned long long nsecs) +{ + u64 end; + + nsecs <<= 9; + do_div(nsecs, 125); + end = get_clock() + nsecs; + if (nsecs & ~0xfffUL) + __udelay(nsecs >> 12); + while (get_clock() < end) + barrier(); +} +EXPORT_SYMBOL(__ndelay); diff --git a/arch/s390/lib/qrnnd.S b/arch/s390/lib/qrnnd.S index eb1df632e749..d321329130ec 100644 --- a/arch/s390/lib/qrnnd.S +++ b/arch/s390/lib/qrnnd.S @@ -1,5 +1,7 @@ # S/390 __udiv_qrnnd +#include <linux/linkage.h> + # r2 : &__r # r3 : upper half of 64 bit word n # r4 : lower half of 64 bit word n @@ -8,8 +10,7 @@ # the quotient q is to be returned .text - .globl __udiv_qrnnd -__udiv_qrnnd: +ENTRY(__udiv_qrnnd) st %r2,24(%r15) # store pointer to reminder for later lr %r0,%r3 # reload n lr %r1,%r4 diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index c66ffd8dbbb7..1f1dba9dcf58 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -91,7 +91,7 @@ static long cmm_alloc_pages(long nr, long *counter, } else free_page((unsigned long) npa); } - diag10(addr); + diag10_range(addr >> PAGE_SHIFT, 1); pa->pages[pa->index++] = addr; (*counter)++; spin_unlock(&cmm_lock); diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index 3cc95dd0a3a6..075ddada4911 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -412,6 +412,7 @@ __segment_load (char *name, int do_nonshared, unsigned long *addr, unsigned long struct dcss_segment *seg; int rc, diag_cc; + start_addr = end_addr = 0; seg = kmalloc(sizeof(*seg), GFP_KERNEL | GFP_DMA); if (seg == NULL) { rc = -ENOMEM; @@ -573,6 +574,7 @@ segment_modify_shared (char *name, int do_nonshared) unsigned long start_addr, end_addr, dummy; int rc, diag_cc; + start_addr = end_addr = 0; mutex_lock(&dcss_lock); seg = segment_by_name (name); if (seg == NULL) { @@ -681,8 +683,6 @@ void segment_save(char *name) { struct dcss_segment *seg; - int startpfn = 0; - int endpfn = 0; char cmd1[160]; char cmd2[80]; int i, response; @@ -698,8 +698,6 @@ segment_save(char *name) goto out; } - startpfn = seg->start_addr >> PAGE_SHIFT; - endpfn = (seg->end) >> PAGE_SHIFT; sprintf(cmd1, "DEFSEG %s", name); for (i=0; i<seg->segcnt; i++) { sprintf(cmd1+strlen(cmd1), " %lX-%lX %s", diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index ab988135e5c6..9564fc779b27 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -34,7 +34,7 @@ #include <asm/asm-offsets.h> #include <asm/system.h> #include <asm/pgtable.h> -#include <asm/s390_ext.h> +#include <asm/irq.h> #include <asm/mmu_context.h> #include <asm/compat.h> #include "../kernel/entry.h" @@ -225,33 +225,6 @@ static noinline void do_sigbus(struct pt_regs *regs, long int_code, force_sig_info(SIGBUS, &si, tsk); } -#ifdef CONFIG_S390_EXEC_PROTECT -static noinline int signal_return(struct pt_regs *regs, long int_code, - unsigned long trans_exc_code) -{ - u16 instruction; - int rc; - - rc = __get_user(instruction, (u16 __user *) regs->psw.addr); - - if (!rc && instruction == 0x0a77) { - clear_tsk_thread_flag(current, TIF_PER_TRAP); - if (is_compat_task()) - sys32_sigreturn(); - else - sys_sigreturn(); - } else if (!rc && instruction == 0x0aad) { - clear_tsk_thread_flag(current, TIF_PER_TRAP); - if (is_compat_task()) - sys32_rt_sigreturn(); - else - sys_rt_sigreturn(); - } else - do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code); - return 0; -} -#endif /* CONFIG_S390_EXEC_PROTECT */ - static noinline void do_fault_error(struct pt_regs *regs, long int_code, unsigned long trans_exc_code, int fault) { @@ -259,13 +232,6 @@ static noinline void do_fault_error(struct pt_regs *regs, long int_code, switch (fault) { case VM_FAULT_BADACCESS: -#ifdef CONFIG_S390_EXEC_PROTECT - if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY && - (trans_exc_code & 3) == 0) { - signal_return(regs, int_code, trans_exc_code); - break; - } -#endif /* CONFIG_S390_EXEC_PROTECT */ case VM_FAULT_BADMAP: /* Bad memory access. Check if it is kernel or user space. */ if (regs->psw.mask & PSW_MASK_PSTATE) { @@ -279,9 +245,12 @@ static noinline void do_fault_error(struct pt_regs *regs, long int_code, do_no_context(regs, int_code, trans_exc_code); break; default: /* fault & VM_FAULT_ERROR */ - if (fault & VM_FAULT_OOM) - pagefault_out_of_memory(); - else if (fault & VM_FAULT_SIGBUS) { + if (fault & VM_FAULT_OOM) { + if (!(regs->psw.mask & PSW_MASK_PSTATE)) + do_no_context(regs, int_code, trans_exc_code); + else + pagefault_out_of_memory(); + } else if (fault & VM_FAULT_SIGBUS) { /* Kernel mode? Handle exceptions or die */ if (!(regs->psw.mask & PSW_MASK_PSTATE)) do_no_context(regs, int_code, trans_exc_code); @@ -311,7 +280,8 @@ static inline int do_exception(struct pt_regs *regs, int access, struct mm_struct *mm; struct vm_area_struct *vma; unsigned long address; - int fault, write; + unsigned int flags; + int fault; if (notify_page_fault(regs)) return 0; @@ -329,9 +299,28 @@ static inline int do_exception(struct pt_regs *regs, int access, goto out; address = trans_exc_code & __FAIL_ADDR_MASK; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + flags = FAULT_FLAG_ALLOW_RETRY; + if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) + flags |= FAULT_FLAG_WRITE; down_read(&mm->mmap_sem); +#ifdef CONFIG_PGSTE + if (test_tsk_thread_flag(current, TIF_SIE) && S390_lowcore.gmap) { + address = gmap_fault(address, + (struct gmap *) S390_lowcore.gmap); + if (address == -EFAULT) { + fault = VM_FAULT_BADMAP; + goto out_up; + } + if (address == -ENOMEM) { + fault = VM_FAULT_OOM; + goto out_up; + } + } +#endif + +retry: fault = VM_FAULT_BADMAP; vma = find_vma(mm, address); if (!vma) @@ -359,21 +348,32 @@ static inline int do_exception(struct pt_regs *regs, int access, * make sure we exit gracefully rather than endlessly redo * the fault. */ - write = (access == VM_WRITE || - (trans_exc_code & store_indication) == 0x400) ? - FAULT_FLAG_WRITE : 0; - fault = handle_mm_fault(mm, vma, address, write); + fault = handle_mm_fault(mm, vma, address, flags); if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + /* + * Major/minor page fault accounting is only done on the + * initial attempt. If we go through a retry, it is extremely + * likely that the page will be found in page cache at that point. + */ + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) { + tsk->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, + regs, address); + } else { + tsk->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, + regs, address); + } + if (fault & VM_FAULT_RETRY) { + /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk + * of starvation. */ + flags &= ~FAULT_FLAG_ALLOW_RETRY; + down_read(&mm->mmap_sem); + goto retry; + } } /* * The instruction that caused the program check will @@ -414,11 +414,6 @@ void __kprobes do_dat_exception(struct pt_regs *regs, long pgm_int_code, int access, fault; access = VM_READ | VM_EXEC | VM_WRITE; -#ifdef CONFIG_S390_EXEC_PROTECT - if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY && - (trans_exc_code & 3) == 0) - access = VM_EXEC; -#endif fault = do_exception(regs, access, trans_exc_code); if (unlikely(fault)) do_fault_error(regs, pgm_int_code & 255, trans_exc_code, fault); @@ -468,10 +463,9 @@ int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write) access = write ? VM_WRITE : VM_READ; fault = do_exception(®s, access, uaddr | 2); if (unlikely(fault)) { - if (fault & VM_FAULT_OOM) { - pagefault_out_of_memory(); - fault = 0; - } else if (fault & VM_FAULT_SIGBUS) + if (fault & VM_FAULT_OOM) + return -EFAULT; + else if (fault & VM_FAULT_SIGBUS) do_sigbus(®s, pgm_int_code, uaddr); } return fault ? -EFAULT : 0; @@ -491,22 +485,28 @@ static int __init nopfault(char *str) __setup("nopfault", nopfault); -typedef struct { - __u16 refdiagc; - __u16 reffcode; - __u16 refdwlen; - __u16 refversn; - __u64 refgaddr; - __u64 refselmk; - __u64 refcmpmk; - __u64 reserved; -} __attribute__ ((packed, aligned(8))) pfault_refbk_t; +struct pfault_refbk { + u16 refdiagc; + u16 reffcode; + u16 refdwlen; + u16 refversn; + u64 refgaddr; + u64 refselmk; + u64 refcmpmk; + u64 reserved; +} __attribute__ ((packed, aligned(8))); int pfault_init(void) { - pfault_refbk_t refbk = - { 0x258, 0, 5, 2, __LC_CURRENT, 1ULL << 48, 1ULL << 48, - __PF_RES_FIELD }; + struct pfault_refbk refbk = { + .refdiagc = 0x258, + .reffcode = 0, + .refdwlen = 5, + .refversn = 2, + .refgaddr = __LC_CURRENT_PID, + .refselmk = 1ULL << 48, + .refcmpmk = 1ULL << 48, + .reserved = __PF_RES_FIELD }; int rc; if (!MACHINE_IS_VM || pfault_disable) @@ -518,18 +518,20 @@ int pfault_init(void) "2:\n" EX_TABLE(0b,1b) : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc"); - __ctl_set_bit(0, 9); return rc; } void pfault_fini(void) { - pfault_refbk_t refbk = - { 0x258, 1, 5, 2, 0ULL, 0ULL, 0ULL, 0ULL }; + struct pfault_refbk refbk = { + .refdiagc = 0x258, + .reffcode = 1, + .refdwlen = 5, + .refversn = 2, + }; if (!MACHINE_IS_VM || pfault_disable) return; - __ctl_clear_bit(0,9); asm volatile( " diag %0,0,0x258\n" "0:\n" @@ -537,11 +539,15 @@ void pfault_fini(void) : : "a" (&refbk), "m" (refbk) : "cc"); } +static DEFINE_SPINLOCK(pfault_lock); +static LIST_HEAD(pfault_list); + static void pfault_interrupt(unsigned int ext_int_code, unsigned int param32, unsigned long param64) { struct task_struct *tsk; __u16 subcode; + pid_t pid; /* * Get the external interruption subcode & pfault @@ -553,44 +559,79 @@ static void pfault_interrupt(unsigned int ext_int_code, if ((subcode & 0xff00) != __SUBCODE_MASK) return; kstat_cpu(smp_processor_id()).irqs[EXTINT_PFL]++; - - /* - * Get the token (= address of the task structure of the affected task). - */ -#ifdef CONFIG_64BIT - tsk = (struct task_struct *) param64; -#else - tsk = (struct task_struct *) param32; -#endif - + if (subcode & 0x0080) { + /* Get the token (= pid of the affected task). */ + pid = sizeof(void *) == 4 ? param32 : param64; + rcu_read_lock(); + tsk = find_task_by_pid_ns(pid, &init_pid_ns); + if (tsk) + get_task_struct(tsk); + rcu_read_unlock(); + if (!tsk) + return; + } else { + tsk = current; + } + spin_lock(&pfault_lock); if (subcode & 0x0080) { /* signal bit is set -> a page has been swapped in by VM */ - if (xchg(&tsk->thread.pfault_wait, -1) != 0) { + if (tsk->thread.pfault_wait == 1) { /* Initial interrupt was faster than the completion * interrupt. pfault_wait is valid. Set pfault_wait * back to zero and wake up the process. This can * safely be done because the task is still sleeping * and can't produce new pfaults. */ tsk->thread.pfault_wait = 0; + list_del(&tsk->thread.list); wake_up_process(tsk); - put_task_struct(tsk); + } else { + /* Completion interrupt was faster than initial + * interrupt. Set pfault_wait to -1 so the initial + * interrupt doesn't put the task to sleep. */ + tsk->thread.pfault_wait = -1; } + put_task_struct(tsk); } else { /* signal bit not set -> a real page is missing. */ - get_task_struct(tsk); - set_task_state(tsk, TASK_UNINTERRUPTIBLE); - if (xchg(&tsk->thread.pfault_wait, 1) != 0) { + if (tsk->thread.pfault_wait == -1) { /* Completion interrupt was faster than the initial - * interrupt (swapped in a -1 for pfault_wait). Set - * pfault_wait back to zero and exit. This can be - * done safely because tsk is running in kernel - * mode and can't produce new pfaults. */ + * interrupt (pfault_wait == -1). Set pfault_wait + * back to zero and exit. */ tsk->thread.pfault_wait = 0; - set_task_state(tsk, TASK_RUNNING); - put_task_struct(tsk); - } else + } else { + /* Initial interrupt arrived before completion + * interrupt. Let the task sleep. */ + tsk->thread.pfault_wait = 1; + list_add(&tsk->thread.list, &pfault_list); + set_task_state(tsk, TASK_UNINTERRUPTIBLE); set_tsk_need_resched(tsk); + } + } + spin_unlock(&pfault_lock); +} + +static int __cpuinit pfault_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + struct thread_struct *thread, *next; + struct task_struct *tsk; + + switch (action) { + case CPU_DEAD: + case CPU_DEAD_FROZEN: + spin_lock_irq(&pfault_lock); + list_for_each_entry_safe(thread, next, &pfault_list, list) { + thread->pfault_wait = 0; + list_del(&thread->list); + tsk = container_of(thread, struct task_struct, thread); + wake_up_process(tsk); + } + spin_unlock_irq(&pfault_lock); + break; + default: + break; } + return NOTIFY_OK; } static int __init pfault_irq_init(void) @@ -599,22 +640,22 @@ static int __init pfault_irq_init(void) if (!MACHINE_IS_VM) return 0; - /* - * Try to get pfault pseudo page faults going. - */ rc = register_external_interrupt(0x2603, pfault_interrupt); - if (rc) { - pfault_disable = 1; - return rc; - } - if (pfault_init() == 0) - return 0; + if (rc) + goto out_extint; + rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP; + if (rc) + goto out_pfault; + service_subclass_irq_register(); + hotcpu_notifier(pfault_cpu_notify, 0); + return 0; - /* Tough luck, no pfault. */ - pfault_disable = 1; +out_pfault: unregister_external_interrupt(0x2603, pfault_interrupt); - return 0; +out_extint: + pfault_disable = 1; + return rc; } early_initcall(pfault_irq_init); -#endif +#endif /* CONFIG_PFAULT */ diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 639cd21f2218..597bb2d27c3c 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -13,7 +13,6 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *pteptr, pte_t pteval) { pmd_t *pmdp = (pmd_t *) pteptr; - pte_t shadow_pteval = pteval; unsigned long mask; if (!MACHINE_HAS_HPAGE) { @@ -21,18 +20,9 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, mask = pte_val(pteval) & (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); pte_val(pteval) = (_SEGMENT_ENTRY + __pa(pteptr)) | mask; - if (mm->context.noexec) { - pteptr += PTRS_PER_PTE; - pte_val(shadow_pteval) = - (_SEGMENT_ENTRY + __pa(pteptr)) | mask; - } } pmd_val(*pmdp) = pte_val(pteval); - if (mm->context.noexec) { - pmdp = get_shadow_table(pmdp); - pmd_val(*pmdp) = pte_val(shadow_pteval); - } } int arch_prepare_hugepage(struct page *page) @@ -45,7 +35,7 @@ int arch_prepare_hugepage(struct page *page) if (MACHINE_HAS_HPAGE) return 0; - ptep = (pte_t *) pte_alloc_one(&init_mm, address); + ptep = (pte_t *) pte_alloc_one(&init_mm, addr); if (!ptep) return -ENOMEM; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index bb409332a484..59b663109d90 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -119,9 +119,7 @@ void __init paging_init(void) sparse_memory_present_with_active_regions(MAX_NUMNODES); sparse_init(); memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); -#ifdef CONFIG_ZONE_DMA max_zone_pfns[ZONE_DMA] = PFN_DOWN(MAX_DMA_ADDRESS); -#endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; free_area_init_nodes(max_zone_pfns); fault_init(); @@ -175,7 +173,8 @@ void kernel_map_pages(struct page *page, int numpages, int enable) pmd = pmd_offset(pud, address); pte = pte_offset_kernel(pmd, address); if (!enable) { - ptep_invalidate(&init_mm, address, pte); + __ptep_ipte(address, pte); + pte_val(*pte) = _PAGE_TYPE_EMPTY; continue; } *pte = mk_pte_phys(address, __pgprot(_PAGE_TYPE_RW)); diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index 71a4b0d34be0..5dbbaa6e594c 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -19,7 +19,7 @@ * using the stura instruction. * Returns the number of bytes copied or -EFAULT. */ -static long probe_kernel_write_odd(void *dst, void *src, size_t size) +static long probe_kernel_write_odd(void *dst, const void *src, size_t size) { unsigned long count, aligned; int offset, mask; @@ -45,7 +45,7 @@ static long probe_kernel_write_odd(void *dst, void *src, size_t size) return rc ? rc : count; } -long probe_kernel_write(void *dst, void *src, size_t size) +long probe_kernel_write(void *dst, const void *src, size_t size) { long copied = 0; @@ -85,3 +85,19 @@ int memcpy_real(void *dest, void *src, size_t count) arch_local_irq_restore(flags); return rc; } + +/* + * Copy memory to absolute zero + */ +void copy_to_absolute_zero(void *dest, void *src, size_t count) +{ + unsigned long cr0; + + BUG_ON((unsigned long) dest + count >= sizeof(struct _lowcore)); + preempt_disable(); + __ctl_store(cr0, 0, 0); + __ctl_clear_bit(0, 28); /* disable lowcore protection */ + memcpy_real(dest + store_prefix(), src, count); + __ctl_load(cr0, 0, 0); + preempt_enable(); +} diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 0607e4b14b27..d013ed39743b 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -28,7 +28,7 @@ static void change_page_attr(unsigned long addr, int numpages, pte = *ptep; pte = set(pte); - ptep_invalidate(&init_mm, addr, ptep); + __ptep_ipte(addr, ptep); *ptep = pte; addr += PAGE_SIZE; } @@ -54,3 +54,8 @@ int set_memory_nx(unsigned long addr, int numpages) return 0; } EXPORT_SYMBOL_GPL(set_memory_nx); + +int set_memory_x(unsigned long addr, int numpages) +{ + return 0; +} diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index e1850c28cd68..4d1f2bce87b3 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -16,6 +16,7 @@ #include <linux/module.h> #include <linux/quicklist.h> #include <linux/rcupdate.h> +#include <linux/slab.h> #include <asm/system.h> #include <asm/pgtable.h> @@ -24,93 +25,12 @@ #include <asm/tlbflush.h> #include <asm/mmu_context.h> -struct rcu_table_freelist { - struct rcu_head rcu; - struct mm_struct *mm; - unsigned int pgt_index; - unsigned int crst_index; - unsigned long *table[0]; -}; - -#define RCU_FREELIST_SIZE \ - ((PAGE_SIZE - sizeof(struct rcu_table_freelist)) \ - / sizeof(unsigned long)) - -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); -static DEFINE_PER_CPU(struct rcu_table_freelist *, rcu_table_freelist); - -static void __page_table_free(struct mm_struct *mm, unsigned long *table); -static void __crst_table_free(struct mm_struct *mm, unsigned long *table); - -static struct rcu_table_freelist *rcu_table_freelist_get(struct mm_struct *mm) -{ - struct rcu_table_freelist **batchp = &__get_cpu_var(rcu_table_freelist); - struct rcu_table_freelist *batch = *batchp; - - if (batch) - return batch; - batch = (struct rcu_table_freelist *) __get_free_page(GFP_ATOMIC); - if (batch) { - batch->mm = mm; - batch->pgt_index = 0; - batch->crst_index = RCU_FREELIST_SIZE; - *batchp = batch; - } - return batch; -} - -static void rcu_table_freelist_callback(struct rcu_head *head) -{ - struct rcu_table_freelist *batch = - container_of(head, struct rcu_table_freelist, rcu); - - while (batch->pgt_index > 0) - __page_table_free(batch->mm, batch->table[--batch->pgt_index]); - while (batch->crst_index < RCU_FREELIST_SIZE) - __crst_table_free(batch->mm, batch->table[batch->crst_index++]); - free_page((unsigned long) batch); -} - -void rcu_table_freelist_finish(void) -{ - struct rcu_table_freelist *batch = __get_cpu_var(rcu_table_freelist); - - if (!batch) - return; - call_rcu(&batch->rcu, rcu_table_freelist_callback); - __get_cpu_var(rcu_table_freelist) = NULL; -} - -static void smp_sync(void *arg) -{ -} - #ifndef CONFIG_64BIT #define ALLOC_ORDER 1 -#define TABLES_PER_PAGE 4 -#define FRAG_MASK 15UL -#define SECOND_HALVES 10UL - -void clear_table_pgstes(unsigned long *table) -{ - clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/4); - memset(table + 256, 0, PAGE_SIZE/4); - clear_table(table + 512, _PAGE_TYPE_EMPTY, PAGE_SIZE/4); - memset(table + 768, 0, PAGE_SIZE/4); -} - +#define FRAG_MASK 0x0f #else #define ALLOC_ORDER 2 -#define TABLES_PER_PAGE 2 -#define FRAG_MASK 3UL -#define SECOND_HALVES 2UL - -void clear_table_pgstes(unsigned long *table) -{ - clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); - memset(table + 256, 0, PAGE_SIZE/2); -} - +#define FRAG_MASK 0x03 #endif unsigned long VMALLOC_START = VMALLOC_END - VMALLOC_SIZE; @@ -125,68 +45,18 @@ static int __init parse_vmalloc(char *arg) } early_param("vmalloc", parse_vmalloc); -unsigned long *crst_table_alloc(struct mm_struct *mm, int noexec) +unsigned long *crst_table_alloc(struct mm_struct *mm) { struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER); if (!page) return NULL; - page->index = 0; - if (noexec) { - struct page *shadow = alloc_pages(GFP_KERNEL, ALLOC_ORDER); - if (!shadow) { - __free_pages(page, ALLOC_ORDER); - return NULL; - } - page->index = page_to_phys(shadow); - } - spin_lock_bh(&mm->context.list_lock); - list_add(&page->lru, &mm->context.crst_list); - spin_unlock_bh(&mm->context.list_lock); return (unsigned long *) page_to_phys(page); } -static void __crst_table_free(struct mm_struct *mm, unsigned long *table) -{ - unsigned long *shadow = get_shadow_table(table); - - if (shadow) - free_pages((unsigned long) shadow, ALLOC_ORDER); - free_pages((unsigned long) table, ALLOC_ORDER); -} - void crst_table_free(struct mm_struct *mm, unsigned long *table) { - struct page *page = virt_to_page(table); - - spin_lock_bh(&mm->context.list_lock); - list_del(&page->lru); - spin_unlock_bh(&mm->context.list_lock); - __crst_table_free(mm, table); -} - -void crst_table_free_rcu(struct mm_struct *mm, unsigned long *table) -{ - struct rcu_table_freelist *batch; - struct page *page = virt_to_page(table); - - spin_lock_bh(&mm->context.list_lock); - list_del(&page->lru); - spin_unlock_bh(&mm->context.list_lock); - if (atomic_read(&mm->mm_users) < 2 && - cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { - __crst_table_free(mm, table); - return; - } - batch = rcu_table_freelist_get(mm); - if (!batch) { - smp_call_function(smp_sync, NULL, 1); - __crst_table_free(mm, table); - return; - } - batch->table[--batch->crst_index] = table; - if (batch->pgt_index >= batch->crst_index) - rcu_table_freelist_finish(); + free_pages((unsigned long) table, ALLOC_ORDER); } #ifdef CONFIG_64BIT @@ -197,7 +67,7 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long limit) BUG_ON(limit > (1UL << 53)); repeat: - table = crst_table_alloc(mm, mm->context.noexec); + table = crst_table_alloc(mm); if (!table) return -ENOMEM; spin_lock_bh(&mm->page_table_lock); @@ -264,140 +134,550 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) } #endif +#ifdef CONFIG_PGSTE + +/** + * gmap_alloc - allocate a guest address space + * @mm: pointer to the parent mm_struct + * + * Returns a guest address space structure. + */ +struct gmap *gmap_alloc(struct mm_struct *mm) +{ + struct gmap *gmap; + struct page *page; + unsigned long *table; + + gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL); + if (!gmap) + goto out; + INIT_LIST_HEAD(&gmap->crst_list); + gmap->mm = mm; + page = alloc_pages(GFP_KERNEL, ALLOC_ORDER); + if (!page) + goto out_free; + list_add(&page->lru, &gmap->crst_list); + table = (unsigned long *) page_to_phys(page); + crst_table_init(table, _REGION1_ENTRY_EMPTY); + gmap->table = table; + list_add(&gmap->list, &mm->context.gmap_list); + return gmap; + +out_free: + kfree(gmap); +out: + return NULL; +} +EXPORT_SYMBOL_GPL(gmap_alloc); + +static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table) +{ + struct gmap_pgtable *mp; + struct gmap_rmap *rmap; + struct page *page; + + if (*table & _SEGMENT_ENTRY_INV) + return 0; + page = pfn_to_page(*table >> PAGE_SHIFT); + mp = (struct gmap_pgtable *) page->index; + list_for_each_entry(rmap, &mp->mapper, list) { + if (rmap->entry != table) + continue; + list_del(&rmap->list); + kfree(rmap); + break; + } + *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; + return 1; +} + +static void gmap_flush_tlb(struct gmap *gmap) +{ + if (MACHINE_HAS_IDTE) + __tlb_flush_idte((unsigned long) gmap->table | + _ASCE_TYPE_REGION1); + else + __tlb_flush_global(); +} + +/** + * gmap_free - free a guest address space + * @gmap: pointer to the guest address space structure + */ +void gmap_free(struct gmap *gmap) +{ + struct page *page, *next; + unsigned long *table; + int i; + + + /* Flush tlb. */ + if (MACHINE_HAS_IDTE) + __tlb_flush_idte((unsigned long) gmap->table | + _ASCE_TYPE_REGION1); + else + __tlb_flush_global(); + + /* Free all segment & region tables. */ + down_read(&gmap->mm->mmap_sem); + list_for_each_entry_safe(page, next, &gmap->crst_list, lru) { + table = (unsigned long *) page_to_phys(page); + if ((*table & _REGION_ENTRY_TYPE_MASK) == 0) + /* Remove gmap rmap structures for segment table. */ + for (i = 0; i < PTRS_PER_PMD; i++, table++) + gmap_unlink_segment(gmap, table); + __free_pages(page, ALLOC_ORDER); + } + up_read(&gmap->mm->mmap_sem); + list_del(&gmap->list); + kfree(gmap); +} +EXPORT_SYMBOL_GPL(gmap_free); + +/** + * gmap_enable - switch primary space to the guest address space + * @gmap: pointer to the guest address space structure + */ +void gmap_enable(struct gmap *gmap) +{ + /* Load primary space page table origin. */ + S390_lowcore.user_asce = _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | __pa(gmap->table); + asm volatile("lctlg 1,1,%0\n" : : "m" (S390_lowcore.user_asce) ); + S390_lowcore.gmap = (unsigned long) gmap; +} +EXPORT_SYMBOL_GPL(gmap_enable); + +/** + * gmap_disable - switch back to the standard primary address space + * @gmap: pointer to the guest address space structure + */ +void gmap_disable(struct gmap *gmap) +{ + /* Load primary space page table origin. */ + S390_lowcore.user_asce = + gmap->mm->context.asce_bits | __pa(gmap->mm->pgd); + asm volatile("lctlg 1,1,%0\n" : : "m" (S390_lowcore.user_asce) ); + S390_lowcore.gmap = 0UL; +} +EXPORT_SYMBOL_GPL(gmap_disable); + +static int gmap_alloc_table(struct gmap *gmap, + unsigned long *table, unsigned long init) +{ + struct page *page; + unsigned long *new; + + page = alloc_pages(GFP_KERNEL, ALLOC_ORDER); + if (!page) + return -ENOMEM; + new = (unsigned long *) page_to_phys(page); + crst_table_init(new, init); + down_read(&gmap->mm->mmap_sem); + if (*table & _REGION_ENTRY_INV) { + list_add(&page->lru, &gmap->crst_list); + *table = (unsigned long) new | _REGION_ENTRY_LENGTH | + (*table & _REGION_ENTRY_TYPE_MASK); + } else + __free_pages(page, ALLOC_ORDER); + up_read(&gmap->mm->mmap_sem); + return 0; +} + +/** + * gmap_unmap_segment - unmap segment from the guest address space + * @gmap: pointer to the guest address space structure + * @addr: address in the guest address space + * @len: length of the memory area to unmap + * + * Returns 0 if the unmap succeded, -EINVAL if not. + */ +int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) +{ + unsigned long *table; + unsigned long off; + int flush; + + if ((to | len) & (PMD_SIZE - 1)) + return -EINVAL; + if (len == 0 || to + len < to) + return -EINVAL; + + flush = 0; + down_read(&gmap->mm->mmap_sem); + for (off = 0; off < len; off += PMD_SIZE) { + /* Walk the guest addr space page table */ + table = gmap->table + (((to + off) >> 53) & 0x7ff); + if (*table & _REGION_ENTRY_INV) + return 0; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 42) & 0x7ff); + if (*table & _REGION_ENTRY_INV) + return 0; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 31) & 0x7ff); + if (*table & _REGION_ENTRY_INV) + return 0; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 20) & 0x7ff); + + /* Clear segment table entry in guest address space. */ + flush |= gmap_unlink_segment(gmap, table); + *table = _SEGMENT_ENTRY_INV; + } + up_read(&gmap->mm->mmap_sem); + if (flush) + gmap_flush_tlb(gmap); + return 0; +} +EXPORT_SYMBOL_GPL(gmap_unmap_segment); + +/** + * gmap_mmap_segment - map a segment to the guest address space + * @gmap: pointer to the guest address space structure + * @from: source address in the parent address space + * @to: target address in the guest address space + * + * Returns 0 if the mmap succeded, -EINVAL or -ENOMEM if not. + */ +int gmap_map_segment(struct gmap *gmap, unsigned long from, + unsigned long to, unsigned long len) +{ + unsigned long *table; + unsigned long off; + int flush; + + if ((from | to | len) & (PMD_SIZE - 1)) + return -EINVAL; + if (len == 0 || from + len > PGDIR_SIZE || + from + len < from || to + len < to) + return -EINVAL; + + flush = 0; + down_read(&gmap->mm->mmap_sem); + for (off = 0; off < len; off += PMD_SIZE) { + /* Walk the gmap address space page table */ + table = gmap->table + (((to + off) >> 53) & 0x7ff); + if ((*table & _REGION_ENTRY_INV) && + gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY)) + goto out_unmap; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 42) & 0x7ff); + if ((*table & _REGION_ENTRY_INV) && + gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY)) + goto out_unmap; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 31) & 0x7ff); + if ((*table & _REGION_ENTRY_INV) && + gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY)) + goto out_unmap; + table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN); + table = table + (((to + off) >> 20) & 0x7ff); + + /* Store 'from' address in an invalid segment table entry. */ + flush |= gmap_unlink_segment(gmap, table); + *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off); + } + up_read(&gmap->mm->mmap_sem); + if (flush) + gmap_flush_tlb(gmap); + return 0; + +out_unmap: + up_read(&gmap->mm->mmap_sem); + gmap_unmap_segment(gmap, to, len); + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(gmap_map_segment); + +unsigned long gmap_fault(unsigned long address, struct gmap *gmap) +{ + unsigned long *table, vmaddr, segment; + struct mm_struct *mm; + struct gmap_pgtable *mp; + struct gmap_rmap *rmap; + struct vm_area_struct *vma; + struct page *page; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + + current->thread.gmap_addr = address; + mm = gmap->mm; + /* Walk the gmap address space page table */ + table = gmap->table + ((address >> 53) & 0x7ff); + if (unlikely(*table & _REGION_ENTRY_INV)) + return -EFAULT; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + ((address >> 42) & 0x7ff); + if (unlikely(*table & _REGION_ENTRY_INV)) + return -EFAULT; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + ((address >> 31) & 0x7ff); + if (unlikely(*table & _REGION_ENTRY_INV)) + return -EFAULT; + table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); + table = table + ((address >> 20) & 0x7ff); + + /* Convert the gmap address to an mm address. */ + segment = *table; + if (likely(!(segment & _SEGMENT_ENTRY_INV))) { + page = pfn_to_page(segment >> PAGE_SHIFT); + mp = (struct gmap_pgtable *) page->index; + return mp->vmaddr | (address & ~PMD_MASK); + } else if (segment & _SEGMENT_ENTRY_RO) { + vmaddr = segment & _SEGMENT_ENTRY_ORIGIN; + vma = find_vma(mm, vmaddr); + if (!vma || vma->vm_start > vmaddr) + return -EFAULT; + + /* Walk the parent mm page table */ + pgd = pgd_offset(mm, vmaddr); + pud = pud_alloc(mm, pgd, vmaddr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, vmaddr); + if (!pmd) + return -ENOMEM; + if (!pmd_present(*pmd) && + __pte_alloc(mm, vma, pmd, vmaddr)) + return -ENOMEM; + /* pmd now points to a valid segment table entry. */ + rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT); + if (!rmap) + return -ENOMEM; + /* Link gmap segment table entry location to page table. */ + page = pmd_page(*pmd); + mp = (struct gmap_pgtable *) page->index; + rmap->entry = table; + list_add(&rmap->list, &mp->mapper); + /* Set gmap segment table entry to page table. */ + *table = pmd_val(*pmd) & PAGE_MASK; + return vmaddr | (address & ~PMD_MASK); + } + return -EFAULT; + +} +EXPORT_SYMBOL_GPL(gmap_fault); + +void gmap_unmap_notifier(struct mm_struct *mm, unsigned long *table) +{ + struct gmap_rmap *rmap, *next; + struct gmap_pgtable *mp; + struct page *page; + int flush; + + flush = 0; + spin_lock(&mm->page_table_lock); + page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + mp = (struct gmap_pgtable *) page->index; + list_for_each_entry_safe(rmap, next, &mp->mapper, list) { + *rmap->entry = + _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; + list_del(&rmap->list); + kfree(rmap); + flush = 1; + } + spin_unlock(&mm->page_table_lock); + if (flush) + __tlb_flush_global(); +} + +static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm, + unsigned long vmaddr) +{ + struct page *page; + unsigned long *table; + struct gmap_pgtable *mp; + + page = alloc_page(GFP_KERNEL|__GFP_REPEAT); + if (!page) + return NULL; + mp = kmalloc(sizeof(*mp), GFP_KERNEL|__GFP_REPEAT); + if (!mp) { + __free_page(page); + return NULL; + } + pgtable_page_ctor(page); + mp->vmaddr = vmaddr & PMD_MASK; + INIT_LIST_HEAD(&mp->mapper); + page->index = (unsigned long) mp; + atomic_set(&page->_mapcount, 3); + table = (unsigned long *) page_to_phys(page); + clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); + clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2); + return table; +} + +static inline void page_table_free_pgste(unsigned long *table) +{ + struct page *page; + struct gmap_pgtable *mp; + + page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + mp = (struct gmap_pgtable *) page->index; + BUG_ON(!list_empty(&mp->mapper)); + pgtable_page_ctor(page); + atomic_set(&page->_mapcount, -1); + kfree(mp); + __free_page(page); +} + +#else /* CONFIG_PGSTE */ + +static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm, + unsigned long vmaddr) +{ + return NULL; +} + +static inline void page_table_free_pgste(unsigned long *table) +{ +} + +static inline void gmap_unmap_notifier(struct mm_struct *mm, + unsigned long *table) +{ +} + +#endif /* CONFIG_PGSTE */ + +static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits) +{ + unsigned int old, new; + + do { + old = atomic_read(v); + new = old ^ bits; + } while (atomic_cmpxchg(v, old, new) != old); + return new; +} + /* * page table entry allocation/free routines. */ -unsigned long *page_table_alloc(struct mm_struct *mm) +unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr) { struct page *page; unsigned long *table; - unsigned long bits; + unsigned int mask, bit; - bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; + if (mm_has_pgste(mm)) + return page_table_alloc_pgste(mm, vmaddr); + /* Allocate fragments of a 4K page as 1K/2K page table */ spin_lock_bh(&mm->context.list_lock); - page = NULL; + mask = FRAG_MASK; if (!list_empty(&mm->context.pgtable_list)) { page = list_first_entry(&mm->context.pgtable_list, struct page, lru); - if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1)) - page = NULL; + table = (unsigned long *) page_to_phys(page); + mask = atomic_read(&page->_mapcount); + mask = mask | (mask >> 4); } - if (!page) { + if ((mask & FRAG_MASK) == FRAG_MASK) { spin_unlock_bh(&mm->context.list_lock); page = alloc_page(GFP_KERNEL|__GFP_REPEAT); if (!page) return NULL; pgtable_page_ctor(page); - page->flags &= ~FRAG_MASK; + atomic_set(&page->_mapcount, 1); table = (unsigned long *) page_to_phys(page); - if (mm->context.has_pgste) - clear_table_pgstes(table); - else - clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); + clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); spin_lock_bh(&mm->context.list_lock); list_add(&page->lru, &mm->context.pgtable_list); + } else { + for (bit = 1; mask & bit; bit <<= 1) + table += PTRS_PER_PTE; + mask = atomic_xor_bits(&page->_mapcount, bit); + if ((mask & FRAG_MASK) == FRAG_MASK) + list_del(&page->lru); } - table = (unsigned long *) page_to_phys(page); - while (page->flags & bits) { - table += 256; - bits <<= 1; - } - page->flags |= bits; - if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1)) - list_move_tail(&page->lru, &mm->context.pgtable_list); spin_unlock_bh(&mm->context.list_lock); return table; } -static void __page_table_free(struct mm_struct *mm, unsigned long *table) +void page_table_free(struct mm_struct *mm, unsigned long *table) { struct page *page; - unsigned long bits; + unsigned int bit, mask; - bits = ((unsigned long) table) & 15; - table = (unsigned long *)(((unsigned long) table) ^ bits); + if (mm_has_pgste(mm)) { + gmap_unmap_notifier(mm, table); + return page_table_free_pgste(table); + } + /* Free 1K/2K page table fragment of a 4K page */ page = pfn_to_page(__pa(table) >> PAGE_SHIFT); - page->flags ^= bits; - if (!(page->flags & FRAG_MASK)) { + bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t))); + spin_lock_bh(&mm->context.list_lock); + if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK) + list_del(&page->lru); + mask = atomic_xor_bits(&page->_mapcount, bit); + if (mask & FRAG_MASK) + list_add(&page->lru, &mm->context.pgtable_list); + spin_unlock_bh(&mm->context.list_lock); + if (mask == 0) { pgtable_page_dtor(page); + atomic_set(&page->_mapcount, -1); __free_page(page); } } -void page_table_free(struct mm_struct *mm, unsigned long *table) +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + +static void __page_table_free_rcu(void *table, unsigned bit) { struct page *page; - unsigned long bits; - bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; - bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long); + if (bit == FRAG_MASK) + return page_table_free_pgste(table); + /* Free 1K/2K page table fragment of a 4K page */ page = pfn_to_page(__pa(table) >> PAGE_SHIFT); - spin_lock_bh(&mm->context.list_lock); - page->flags ^= bits; - if (page->flags & FRAG_MASK) { - /* Page now has some free pgtable fragments. */ - if (!list_empty(&page->lru)) - list_move(&page->lru, &mm->context.pgtable_list); - page = NULL; - } else - /* All fragments of the 4K page have been freed. */ - list_del(&page->lru); - spin_unlock_bh(&mm->context.list_lock); - if (page) { + if (atomic_xor_bits(&page->_mapcount, bit) == 0) { pgtable_page_dtor(page); + atomic_set(&page->_mapcount, -1); __free_page(page); } } -void page_table_free_rcu(struct mm_struct *mm, unsigned long *table) +void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table) { - struct rcu_table_freelist *batch; + struct mm_struct *mm; struct page *page; - unsigned long bits; + unsigned int bit, mask; - if (atomic_read(&mm->mm_users) < 2 && - cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { - page_table_free(mm, table); - return; - } - batch = rcu_table_freelist_get(mm); - if (!batch) { - smp_call_function(smp_sync, NULL, 1); - page_table_free(mm, table); + mm = tlb->mm; + if (mm_has_pgste(mm)) { + gmap_unmap_notifier(mm, table); + table = (unsigned long *) (__pa(table) | FRAG_MASK); + tlb_remove_table(tlb, table); return; } - bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; - bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long); + bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t))); page = pfn_to_page(__pa(table) >> PAGE_SHIFT); spin_lock_bh(&mm->context.list_lock); - /* Delayed freeing with rcu prevents reuse of pgtable fragments */ - list_del_init(&page->lru); + if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK) + list_del(&page->lru); + mask = atomic_xor_bits(&page->_mapcount, bit | (bit << 4)); + if (mask & FRAG_MASK) + list_add_tail(&page->lru, &mm->context.pgtable_list); spin_unlock_bh(&mm->context.list_lock); - table = (unsigned long *)(((unsigned long) table) | bits); - batch->table[batch->pgt_index++] = table; - if (batch->pgt_index >= batch->crst_index) - rcu_table_freelist_finish(); + table = (unsigned long *) (__pa(table) | (bit << 4)); + tlb_remove_table(tlb, table); } -void disable_noexec(struct mm_struct *mm, struct task_struct *tsk) +void __tlb_remove_table(void *_table) { - struct page *page; + void *table = (void *)((unsigned long) _table & PAGE_MASK); + unsigned type = (unsigned long) _table & ~PAGE_MASK; - spin_lock_bh(&mm->context.list_lock); - /* Free shadow region and segment tables. */ - list_for_each_entry(page, &mm->context.crst_list, lru) - if (page->index) { - free_pages((unsigned long) page->index, ALLOC_ORDER); - page->index = 0; - } - /* "Free" second halves of page tables. */ - list_for_each_entry(page, &mm->context.pgtable_list, lru) - page->flags &= ~SECOND_HALVES; - spin_unlock_bh(&mm->context.list_lock); - mm->context.noexec = 0; - update_mm(mm, tsk); + if (type) + __page_table_free_rcu(table, type); + else + free_pages((unsigned long) table, ALLOC_ORDER); } +#endif + /* * switch on pgstes for its userspace process (for kvm) */ @@ -411,7 +691,7 @@ int s390_enable_sie(void) return -EINVAL; /* Do we have pgstes? if yes, we are done */ - if (tsk->mm->context.has_pgste) + if (mm_has_pgste(tsk->mm)) return 0; /* lets check if we are allowed to replace the mm */ diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 34c43f23b28c..781ff5169560 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -61,12 +61,12 @@ static inline pmd_t *vmem_pmd_alloc(void) return pmd; } -static pte_t __ref *vmem_pte_alloc(void) +static pte_t __ref *vmem_pte_alloc(unsigned long address) { pte_t *pte; if (slab_is_available()) - pte = (pte_t *) page_table_alloc(&init_mm); + pte = (pte_t *) page_table_alloc(&init_mm, address); else pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); if (!pte) @@ -95,7 +95,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) pu_dir = vmem_pud_alloc(); if (!pu_dir) goto out; - pgd_populate_kernel(&init_mm, pg_dir, pu_dir); + pgd_populate(&init_mm, pg_dir, pu_dir); } pu_dir = pud_offset(pg_dir, address); @@ -103,7 +103,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) pm_dir = vmem_pmd_alloc(); if (!pm_dir) goto out; - pud_populate_kernel(&init_mm, pu_dir, pm_dir); + pud_populate(&init_mm, pu_dir, pm_dir); } pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0)); @@ -120,10 +120,10 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) } #endif if (pmd_none(*pm_dir)) { - pt_dir = vmem_pte_alloc(); + pt_dir = vmem_pte_alloc(address); if (!pt_dir) goto out; - pmd_populate_kernel(&init_mm, pm_dir, pt_dir); + pmd_populate(&init_mm, pm_dir, pt_dir); } pt_dir = pte_offset_kernel(pm_dir, address); @@ -159,7 +159,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size) continue; if (pmd_huge(*pm_dir)) { - pmd_clear_kernel(pm_dir); + pmd_clear(pm_dir); address += HPAGE_SIZE - PAGE_SIZE; continue; } @@ -192,7 +192,7 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) pu_dir = vmem_pud_alloc(); if (!pu_dir) goto out; - pgd_populate_kernel(&init_mm, pg_dir, pu_dir); + pgd_populate(&init_mm, pg_dir, pu_dir); } pu_dir = pud_offset(pg_dir, address); @@ -200,15 +200,15 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) pm_dir = vmem_pmd_alloc(); if (!pm_dir) goto out; - pud_populate_kernel(&init_mm, pu_dir, pm_dir); + pud_populate(&init_mm, pu_dir, pm_dir); } pm_dir = pmd_offset(pu_dir, address); if (pmd_none(*pm_dir)) { - pt_dir = vmem_pte_alloc(); + pt_dir = vmem_pte_alloc(address); if (!pt_dir) goto out; - pmd_populate_kernel(&init_mm, pm_dir, pt_dir); + pmd_populate(&init_mm, pm_dir, pt_dir); } pt_dir = pte_offset_kernel(pm_dir, address); diff --git a/arch/s390/oprofile/hwsampler.c b/arch/s390/oprofile/hwsampler.c index 4952872d6f0a..4552ce40c81a 100644 --- a/arch/s390/oprofile/hwsampler.c +++ b/arch/s390/oprofile/hwsampler.c @@ -5,6 +5,7 @@ * Author: Heinz Graalfs <graalfs@de.ibm.com> */ +#include <linux/kernel_stat.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/smp.h> @@ -18,7 +19,7 @@ #include <linux/oprofile.h> #include <asm/lowcore.h> -#include <asm/s390_ext.h> +#include <asm/irq.h> #include "hwsampler.h" @@ -579,7 +580,7 @@ static int hws_cpu_callback(struct notifier_block *nfb, { /* We do not have sampler space available for all possible CPUs. All CPUs should be online when hw sampling is activated. */ - return NOTIFY_BAD; + return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD; } static struct notifier_block hws_cpu_notifier = { @@ -674,17 +675,11 @@ int hwsampler_activate(unsigned int cpu) static void hws_ext_handler(unsigned int ext_int_code, unsigned int param32, unsigned long param64) { - int cpu; struct hws_cpu_buffer *cb; - cpu = smp_processor_id(); - cb = &per_cpu(sampler_cpu_buffer, cpu); - - atomic_xchg( - &cb->ext_params, - atomic_read(&cb->ext_params) - | S390_lowcore.ext_params); - + kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++; + cb = &__get_cpu_var(sampler_cpu_buffer); + atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32); if (hws_wq) queue_work(hws_wq, &cb->worker); } @@ -764,7 +759,7 @@ static int worker_check_error(unsigned int cpu, int ext_params) if (!sdbt || !*sdbt) return -EINVAL; - if (ext_params & EI_IEA) + if (ext_params & EI_PRA) cb->req_alert++; if (ext_params & EI_LSDA) @@ -1009,7 +1004,7 @@ int hwsampler_deallocate() if (hws_state != HWS_STOPPED) goto deallocate_exit; - smp_ctl_clear_bit(0, 5); /* set bit 58 CR0 off */ + ctl_clear_bit(0, 5); /* set bit 58 CR0 off */ deallocate_sdbt(); hws_state = HWS_DEALLOCATED; @@ -1021,20 +1016,14 @@ deallocate_exit: return rc; } -long hwsampler_query_min_interval(void) +unsigned long hwsampler_query_min_interval(void) { - if (min_sampler_rate) - return min_sampler_rate; - else - return -EINVAL; + return min_sampler_rate; } -long hwsampler_query_max_interval(void) +unsigned long hwsampler_query_max_interval(void) { - if (max_sampler_rate) - return max_sampler_rate; - else - return -EINVAL; + return max_sampler_rate; } unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu) @@ -1129,7 +1118,7 @@ int hwsampler_shutdown() mutex_lock(&hws_sem); if (hws_state == HWS_STOPPED) { - smp_ctl_clear_bit(0, 5); /* set bit 58 CR0 off */ + ctl_clear_bit(0, 5); /* set bit 58 CR0 off */ deallocate_sdbt(); } if (hws_wq) { @@ -1204,7 +1193,7 @@ start_all_exit: hws_oom = 1; hws_flush_all = 0; /* now let them in, 1407 CPUMF external interrupts */ - smp_ctl_set_bit(0, 5); /* set CR0 bit 58 */ + ctl_set_bit(0, 5); /* set CR0 bit 58 */ return 0; } diff --git a/arch/s390/oprofile/hwsampler.h b/arch/s390/oprofile/hwsampler.h index 8c72b59316b5..1912f3bb190c 100644 --- a/arch/s390/oprofile/hwsampler.h +++ b/arch/s390/oprofile/hwsampler.h @@ -102,8 +102,8 @@ int hwsampler_setup(void); int hwsampler_shutdown(void); int hwsampler_allocate(unsigned long sdbt, unsigned long sdb); int hwsampler_deallocate(void); -long hwsampler_query_min_interval(void); -long hwsampler_query_max_interval(void); +unsigned long hwsampler_query_min_interval(void); +unsigned long hwsampler_query_max_interval(void); int hwsampler_start_all(unsigned long interval); int hwsampler_stop_all(void); int hwsampler_deactivate(unsigned int cpu); diff --git a/arch/s390/oprofile/init.c b/arch/s390/oprofile/init.c index c63d7e58352b..6efc18b5e60a 100644 --- a/arch/s390/oprofile/init.c +++ b/arch/s390/oprofile/init.c @@ -13,8 +13,6 @@ #include <linux/oprofile.h> #include <linux/init.h> #include <linux/errno.h> -#include <linux/oprofile.h> -#include <linux/errno.h> #include <linux/fs.h> #include "../../../drivers/oprofile/oprof.h" @@ -25,7 +23,7 @@ extern void s390_backtrace(struct pt_regs * const regs, unsigned int depth); #include "hwsampler.h" -#define DEFAULT_INTERVAL 4096 +#define DEFAULT_INTERVAL 4127518 #define DEFAULT_SDBT_BLOCKS 1 #define DEFAULT_SDB_BLOCKS 511 @@ -145,15 +143,17 @@ static int oprofile_hwsampler_init(struct oprofile_operations *ops) * create hwsampler files only if hwsampler_setup() succeeds. */ oprofile_min_interval = hwsampler_query_min_interval(); - if (oprofile_min_interval < 0) { - oprofile_min_interval = 0; + if (oprofile_min_interval == 0) return -ENODEV; - } oprofile_max_interval = hwsampler_query_max_interval(); - if (oprofile_max_interval < 0) { - oprofile_max_interval = 0; + if (oprofile_max_interval == 0) return -ENODEV; - } + + /* The initial value should be sane */ + if (oprofile_hw_interval < oprofile_min_interval) + oprofile_hw_interval = oprofile_min_interval; + if (oprofile_hw_interval > oprofile_max_interval) + oprofile_hw_interval = oprofile_max_interval; if (oprofile_timer_init(ops)) return -ENODEV; |