diff options
Diffstat (limited to 'drivers/s390/crypto')
30 files changed, 6533 insertions, 4157 deletions
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile index 0edacd101c12..e83c6603c858 100644 --- a/drivers/s390/crypto/Makefile +++ b/drivers/s390/crypto/Makefile @@ -4,7 +4,7 @@ # ap-objs := ap_bus.o ap_card.o ap_queue.o -obj-$(subst m,y,$(CONFIG_ZCRYPT)) += ap.o +obj-$(CONFIG_AP) += ap.o # zcrypt_api.o and zcrypt_msgtype*.o depend on ap.o zcrypt-objs := zcrypt_api.o zcrypt_card.o zcrypt_queue.o zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o @@ -13,10 +13,26 @@ obj-$(CONFIG_ZCRYPT) += zcrypt.o # adapter drivers depend on ap.o and zcrypt.o obj-$(CONFIG_ZCRYPT) += zcrypt_cex4.o -# pkey kernel module -pkey-objs := pkey_api.o +# pkey base and api module +pkey-objs := pkey_base.o pkey_api.o pkey_sysfs.o obj-$(CONFIG_PKEY) += pkey.o +# pkey cca handler module +pkey-cca-objs := pkey_cca.o +obj-$(CONFIG_PKEY_CCA) += pkey-cca.o + +# pkey ep11 handler module +pkey-ep11-objs := pkey_ep11.o +obj-$(CONFIG_PKEY_EP11) += pkey-ep11.o + +# pkey pckmo handler module +pkey-pckmo-objs := pkey_pckmo.o +obj-$(CONFIG_PKEY_PCKMO) += pkey-pckmo.o + +# pkey uv handler module +pkey-uv-objs := pkey_uv.o +obj-$(CONFIG_PKEY_UV) += pkey-uv.o + # adjunct processor matrix vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o obj-$(CONFIG_VFIO_AP) += vfio_ap.o diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 339812efe822..a445494fd2be 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -11,11 +11,11 @@ * Adjunct processor bus. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt #include <linux/kernel_stat.h> #include <linux/moduleparam.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/delay.h> #include <linux/err.h> @@ -26,6 +26,7 @@ #include <linux/notifier.h> #include <linux/kthread.h> #include <linux/mutex.h> +#include <asm/machine.h> #include <asm/airq.h> #include <asm/tpi.h> #include <linux/atomic.h> @@ -38,13 +39,17 @@ #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/module.h> +#include <asm/uv.h> +#include <asm/chsc.h> +#include <linux/mempool.h> #include "ap_bus.h" #include "ap_debug.h" -/* - * Module parameters; note though this file itself isn't modular. - */ +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("Adjunct Processor Bus driver"); +MODULE_LICENSE("GPL"); + int ap_domain_index = -1; /* Adjunct Processor Domain Index */ static DEFINE_SPINLOCK(ap_domain_lock); module_param_named(domain, ap_domain_index, int, 0440); @@ -80,20 +85,27 @@ DEFINE_SPINLOCK(ap_queues_lock); /* Default permissions (ioctl, card and domain masking) */ struct ap_perms ap_perms; EXPORT_SYMBOL(ap_perms); -DEFINE_MUTEX(ap_perms_mutex); -EXPORT_SYMBOL(ap_perms_mutex); - -/* # of bus scans since init */ -static atomic64_t ap_scan_bus_count; +/* true if apmask and/or aqmask are NOT default */ +bool ap_apmask_aqmask_in_use; +/* counter for how many driver_overrides are currently active */ +int ap_driver_override_ctr; +/* + * Mutex for consistent read and write of the ap_perms struct, + * ap_apmask_aqmask_in_use, ap_driver_override_ctr + * and the ap bus sysfs attributes apmask and aqmask. + */ +DEFINE_MUTEX(ap_attr_mutex); +EXPORT_SYMBOL(ap_attr_mutex); /* # of bindings complete since init */ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); -/* completion for initial APQN bindings complete */ -static DECLARE_COMPLETION(ap_init_apqn_bindings_complete); +/* completion for APQN bindings complete */ +static DECLARE_COMPLETION(ap_apqn_bindings_complete); -static struct ap_config_info *ap_qci_info; -static struct ap_config_info *ap_qci_info_old; +static struct ap_config_info qci[2]; +static struct ap_config_info *const ap_qci_info = &qci[0]; +static struct ap_config_info *const ap_qci_info_old = &qci[1]; /* * AP bus related debug feature things. @@ -101,12 +113,38 @@ static struct ap_config_info *ap_qci_info_old; debug_info_t *ap_dbf_info; /* - * Workqueue timer for bus rescan. + * There is a need for a do-not-allocate-memory path through the AP bus + * layer. The pkey layer may be triggered via the in-kernel interface from + * a protected key crypto algorithm (namely PAES) to convert a secure key + * into a protected key. This happens in a workqueue context, so sleeping + * is allowed but memory allocations causing IO operations are not permitted. + * To accomplish this, an AP message memory pool with pre-allocated space + * is established. When ap_init_apmsg() with use_mempool set to true is + * called, instead of kmalloc() the ap message buffer is allocated from + * the ap_msg_pool. This pool only holds a limited amount of buffers: + * ap_msg_pool_min_items with the item size AP_DEFAULT_MAX_MSG_SIZE and + * exactly one of these items (if available) is returned if ap_init_apmsg() + * with the use_mempool arg set to true is called. When this pool is exhausted + * and use_mempool is set true, ap_init_apmsg() returns -ENOMEM without + * any attempt to allocate memory and the caller has to deal with that. + */ +static mempool_t *ap_msg_pool; +static unsigned int ap_msg_pool_min_items = 8; +module_param_named(msgpool_min_items, ap_msg_pool_min_items, uint, 0440); +MODULE_PARM_DESC(msgpool_min_items, "AP message pool minimal items"); + +/* + * AP bus rescan related things. */ -static struct timer_list ap_config_timer; -static int ap_config_time = AP_CONFIG_TIME; -static void ap_scan_bus(struct work_struct *); -static DECLARE_WORK(ap_scan_work, ap_scan_bus); +static bool ap_scan_bus(void); +static bool ap_scan_bus_result; /* result of last ap_scan_bus() */ +static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */ +static struct task_struct *ap_scan_bus_task; /* thread holding the scan mutex */ +static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ +static int ap_scan_bus_time = AP_CONFIG_TIME; +static struct timer_list ap_scan_bus_timer; +static void ap_scan_bus_wq_callback(struct work_struct *); +static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus_wq_callback); /* * Tasklet & timer for AP request polling and interrupts @@ -135,7 +173,7 @@ static int ap_max_domain_id = 15; /* Maximum adapter id, if not given via qci */ static int ap_max_adapter_id = 63; -static struct bus_type ap_bus_type; +static const struct bus_type ap_bus_type; /* Adapter interrupt definitions */ static void ap_interrupt_handler(struct airq_struct *airq, @@ -201,9 +239,7 @@ static int ap_apft_available(void) */ static inline int ap_qact_available(void) { - if (ap_qci_info) - return ap_qci_info->qact; - return 0; + return ap_qci_info->qact; } /* @@ -213,9 +249,7 @@ static inline int ap_qact_available(void) */ int ap_sb_available(void) { - if (ap_qci_info) - return ap_qci_info->apsb; - return 0; + return ap_qci_info->apsb; } /* @@ -227,23 +261,6 @@ bool ap_is_se_guest(void) } EXPORT_SYMBOL(ap_is_se_guest); -/* - * ap_fetch_qci_info(): Fetch cryptographic config info - * - * Returns the ap configuration info fetched via PQAP(QCI). - * On success 0 is returned, on failure a negative errno - * is returned, e.g. if the PQAP(QCI) instruction is not - * available, the return value will be -EOPNOTSUPP. - */ -static inline int ap_fetch_qci_info(struct ap_config_info *info) -{ - if (!ap_qci_available()) - return -EOPNOTSUPP; - if (!info) - return -EINVAL; - return ap_qci(info); -} - /** * ap_init_qci_info(): Allocate and query qci config info. * Does also update the static variables ap_max_domain_id @@ -251,27 +268,12 @@ static inline int ap_fetch_qci_info(struct ap_config_info *info) */ static void __init ap_init_qci_info(void) { - if (!ap_qci_available()) { + if (!ap_qci_available() || + ap_qci(ap_qci_info)) { AP_DBF_INFO("%s QCI not supported\n", __func__); return; } - - ap_qci_info = kzalloc(sizeof(*ap_qci_info), GFP_KERNEL); - if (!ap_qci_info) - return; - ap_qci_info_old = kzalloc(sizeof(*ap_qci_info_old), GFP_KERNEL); - if (!ap_qci_info_old) { - kfree(ap_qci_info); - ap_qci_info = NULL; - return; - } - if (ap_fetch_qci_info(ap_qci_info) != 0) { - kfree(ap_qci_info); - kfree(ap_qci_info_old); - ap_qci_info = NULL; - ap_qci_info_old = NULL; - return; - } + memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); AP_DBF_INFO("%s successful fetched initial qci info\n", __func__); if (ap_qci_info->apxa) { @@ -286,8 +288,6 @@ static void __init ap_init_qci_info(void) __func__, ap_max_domain_id); } } - - memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); } /* @@ -310,7 +310,7 @@ static inline int ap_test_config_card_id(unsigned int id) { if (id > ap_max_adapter_id) return 0; - if (ap_qci_info) + if (ap_qci_info->flags) return ap_test_config(ap_qci_info->apm, id); return 1; } @@ -327,7 +327,7 @@ int ap_test_config_usage_domain(unsigned int domain) { if (domain > ap_max_domain_id) return 0; - if (ap_qci_info) + if (ap_qci_info->flags) return ap_test_config(ap_qci_info->aqm, domain); return 1; } @@ -352,18 +352,17 @@ EXPORT_SYMBOL(ap_test_config_ctrl_domain); /* * ap_queue_info(): Check and get AP queue info. * Returns: 1 if APQN exists and info is filled, - * 0 if APQN seems to exit but there is no info + * 0 if APQN seems to exist but there is no info * available (eg. caused by an asynch pending error) * -1 invalid APQN, TAPQ error or AP queue status which * indicates there is no APQN. */ -static int ap_queue_info(ap_qid_t qid, int *q_type, unsigned int *q_fac, - int *q_depth, int *q_ml, bool *q_decfg, bool *q_cstop) +static int ap_queue_info(ap_qid_t qid, struct ap_tapq_hwinfo *hwinfo, + bool *decfg, bool *cstop) { struct ap_queue_status status; - struct ap_tapq_gr2 tapq_info; - tapq_info.value = 0; + hwinfo->value = 0; /* make sure we don't run into a specifiation exception */ if (AP_QID_CARD(qid) > ap_max_adapter_id || @@ -371,11 +370,7 @@ static int ap_queue_info(ap_qid_t qid, int *q_type, unsigned int *q_fac, return -1; /* call TAPQ on this APQN */ - status = ap_test_queue(qid, ap_apft_available(), &tapq_info); - - /* handle pending async error with return 'no info available' */ - if (status.async) - return 0; + status = ap_test_queue(qid, ap_apft_available(), hwinfo); switch (status.response_code) { case AP_RESPONSE_NORMAL: @@ -383,26 +378,23 @@ static int ap_queue_info(ap_qid_t qid, int *q_type, unsigned int *q_fac, case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: case AP_RESPONSE_BUSY: - /* - * According to the architecture in all these cases the - * info should be filled. All bits 0 is not possible as - * there is at least one of the mode bits set. - */ - if (WARN_ON_ONCE(!tapq_info.value)) - return 0; - *q_type = tapq_info.at; - *q_fac = tapq_info.fac; - *q_depth = tapq_info.qd; - *q_ml = tapq_info.ml; - *q_decfg = status.response_code == AP_RESPONSE_DECONFIGURED; - *q_cstop = status.response_code == AP_RESPONSE_CHECKSTOPPED; - return 1; + /* For all these RCs the tapq info should be available */ + break; default: - /* - * A response code which indicates, there is no info available. - */ - return -1; + /* On a pending async error the info should be available */ + if (!status.async) + return -1; + break; } + + /* There should be at least one of the mode bits set */ + if (WARN_ON_ONCE(!hwinfo->value)) + return 0; + + *decfg = status.response_code == AP_RESPONSE_DECONFIGURED; + *cstop = status.response_code == AP_RESPONSE_CHECKSTOPPED; + + return 1; } void ap_wait(enum ap_sm_wait wait) @@ -445,7 +437,7 @@ void ap_wait(enum ap_sm_wait wait) */ void ap_request_timeout(struct timer_list *t) { - struct ap_queue *aq = from_timer(aq, t, timeout); + struct ap_queue *aq = timer_container_of(aq, t, timeout); spin_lock_bh(&aq->lock); ap_wait(ap_sm_event(aq, AP_SM_EVENT_TIMEOUT)); @@ -493,7 +485,7 @@ static void ap_tasklet_fn(unsigned long dummy) * important that no requests on any AP get lost. */ if (ap_irq_flag) - xchg(ap_airq.lsi_ptr, 0); + WRITE_ONCE(*ap_airq.lsi_ptr, 0); spin_lock_bh(&ap_queues_lock); hash_for_each(ap_queues, bkt, aq, hnode) { @@ -586,6 +578,48 @@ static void ap_poll_thread_stop(void) #define is_card_dev(x) ((x)->parent == ap_root_device) #define is_queue_dev(x) ((x)->parent != ap_root_device) +/* + * ap_init_apmsg() - Initialize ap_message. + */ +int ap_init_apmsg(struct ap_message *ap_msg, u32 flags) +{ + unsigned int maxmsgsize; + + memset(ap_msg, 0, sizeof(*ap_msg)); + ap_msg->flags = flags; + + if (flags & AP_MSG_FLAG_MEMPOOL) { + ap_msg->msg = mempool_alloc_preallocated(ap_msg_pool); + if (!ap_msg->msg) + return -ENOMEM; + ap_msg->bufsize = AP_DEFAULT_MAX_MSG_SIZE; + return 0; + } + + maxmsgsize = atomic_read(&ap_max_msg_size); + ap_msg->msg = kmalloc(maxmsgsize, GFP_KERNEL); + if (!ap_msg->msg) + return -ENOMEM; + ap_msg->bufsize = maxmsgsize; + + return 0; +} +EXPORT_SYMBOL(ap_init_apmsg); + +/* + * ap_release_apmsg() - Release ap_message. + */ +void ap_release_apmsg(struct ap_message *ap_msg) +{ + if (ap_msg->flags & AP_MSG_FLAG_MEMPOOL) { + memzero_explicit(ap_msg->msg, ap_msg->bufsize); + mempool_free(ap_msg->msg, ap_msg_pool); + } else { + kfree_sensitive(ap_msg->msg); + } +} +EXPORT_SYMBOL(ap_release_apmsg); + /** * ap_bus_match() * @dev: Pointer to device @@ -593,9 +627,9 @@ static void ap_poll_thread_stop(void) * * AP bus driver registration/unregistration. */ -static int ap_bus_match(struct device *dev, struct device_driver *drv) +static int ap_bus_match(struct device *dev, const struct device_driver *drv) { - struct ap_driver *ap_drv = to_ap_drv(drv); + const struct ap_driver *ap_drv = to_ap_drv(drv); struct ap_device_id *id; /* @@ -645,11 +679,11 @@ static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env) return rc; /* Add MODE=<accel|cca|ep11> */ - if (ap_test_bit(&ac->functions, AP_FUNC_ACCEL)) + if (ac->hwinfo.accel) rc = add_uevent_var(env, "MODE=accel"); - else if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) + else if (ac->hwinfo.cca) rc = add_uevent_var(env, "MODE=cca"); - else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) + else if (ac->hwinfo.ep11) rc = add_uevent_var(env, "MODE=ep11"); if (rc) return rc; @@ -657,11 +691,11 @@ static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env) struct ap_queue *aq = to_ap_queue(&ap_dev->device); /* Add MODE=<accel|cca|ep11> */ - if (ap_test_bit(&aq->card->functions, AP_FUNC_ACCEL)) + if (aq->card->hwinfo.accel) rc = add_uevent_var(env, "MODE=accel"); - else if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) + else if (aq->card->hwinfo.cca) rc = add_uevent_var(env, "MODE=cca"); - else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) + else if (aq->card->hwinfo.ep11) rc = add_uevent_var(env, "MODE=ep11"); if (rc) return rc; @@ -761,7 +795,7 @@ static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound) } /* - * After initial ap bus scan do check if all existing APQNs are + * After ap bus scan do check if all existing APQNs are * bound to device drivers. */ static void ap_check_bindings_complete(void) @@ -771,11 +805,11 @@ static void ap_check_bindings_complete(void) if (atomic64_read(&ap_scan_bus_count) >= 1) { ap_calc_bound_apqns(&apqns, &bound); if (bound == apqns) { - if (!completion_done(&ap_init_apqn_bindings_complete)) { - complete_all(&ap_init_apqn_bindings_complete); - AP_DBF_INFO("%s complete\n", __func__); + if (!completion_done(&ap_apqn_bindings_complete)) { + complete_all(&ap_apqn_bindings_complete); + ap_send_bindings_complete_uevent(); + pr_debug("all apqn bindings complete\n"); } - ap_send_bindings_complete_uevent(); } } } @@ -790,27 +824,29 @@ static void ap_check_bindings_complete(void) * -ETIME is returned. On failures negative return values are * returned to the caller. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout) +int ap_wait_apqn_bindings_complete(unsigned long timeout) { + int rc = 0; long l; - if (completion_done(&ap_init_apqn_bindings_complete)) + if (completion_done(&ap_apqn_bindings_complete)) return 0; if (timeout) l = wait_for_completion_interruptible_timeout( - &ap_init_apqn_bindings_complete, timeout); + &ap_apqn_bindings_complete, timeout); else l = wait_for_completion_interruptible( - &ap_init_apqn_bindings_complete); + &ap_apqn_bindings_complete); if (l < 0) - return l == -ERESTARTSYS ? -EINTR : l; + rc = l == -ERESTARTSYS ? -EINTR : l; else if (l == 0 && timeout) - return -ETIME; + rc = -ETIME; - return 0; + pr_debug("rc=%d\n", rc); + return rc; } -EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete); +EXPORT_SYMBOL(ap_wait_apqn_bindings_complete); static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) { @@ -825,21 +861,38 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) int rc, card, queue, devres, drvres; if (is_queue_dev(dev)) { - card = AP_QID_CARD(to_ap_queue(dev)->qid); - queue = AP_QID_QUEUE(to_ap_queue(dev)->qid); - mutex_lock(&ap_perms_mutex); - devres = test_bit_inv(card, ap_perms.apm) && - test_bit_inv(queue, ap_perms.aqm); - mutex_unlock(&ap_perms_mutex); - drvres = to_ap_drv(dev->driver)->flags - & AP_DRIVER_FLAG_DEFAULT; - if (!!devres != !!drvres) { - AP_DBF_DBG("%s reprobing queue=%02x.%04x\n", - __func__, card, queue); - rc = device_reprobe(dev); - if (rc) - AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", - __func__, card, queue); + struct ap_driver *ap_drv = to_ap_drv(dev->driver); + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + + card = AP_QID_CARD(aq->qid); + queue = AP_QID_QUEUE(aq->qid); + + if (ap_dev->driver_override) { + if (strcmp(ap_dev->driver_override, + ap_drv->driver.name)) { + pr_debug("reprobing queue=%02x.%04x\n", card, queue); + rc = device_reprobe(dev); + if (rc) { + AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", + __func__, card, queue); + } + } + } else { + mutex_lock(&ap_attr_mutex); + devres = test_bit_inv(card, ap_perms.apm) && + test_bit_inv(queue, ap_perms.aqm); + mutex_unlock(&ap_attr_mutex); + drvres = to_ap_drv(dev->driver)->flags + & AP_DRIVER_FLAG_DEFAULT; + if (!!devres != !!drvres) { + pr_debug("reprobing queue=%02x.%04x\n", card, queue); + rc = device_reprobe(dev); + if (rc) { + AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", + __func__, card, queue); + } + } } } @@ -857,22 +910,37 @@ static void ap_bus_revise_bindings(void) * @card: the APID of the adapter card to check * @queue: the APQI of the queue to check * - * Note: the ap_perms_mutex must be locked by the caller of this function. + * Note: the ap_attr_mutex must be locked by the caller of this function. * * Return: an int specifying whether the AP adapter is reserved for the host (1) * or not (0). */ int ap_owned_by_def_drv(int card, int queue) { + struct ap_queue *aq; int rc = 0; if (card < 0 || card >= AP_DEVICES || queue < 0 || queue >= AP_DOMAINS) return -EINVAL; + aq = ap_get_qdev(AP_MKQID(card, queue)); + if (aq) { + const struct device_driver *drv = aq->ap_dev.device.driver; + const struct ap_driver *ap_drv = to_ap_drv(drv); + bool override = !!aq->ap_dev.driver_override; + + if (override && drv && ap_drv->flags & AP_DRIVER_FLAG_DEFAULT) + rc = 1; + put_device(&aq->ap_dev.device); + if (override) + goto out; + } + if (test_bit_inv(card, ap_perms.apm) && test_bit_inv(queue, ap_perms.aqm)) rc = 1; +out: return rc; } EXPORT_SYMBOL(ap_owned_by_def_drv); @@ -884,7 +952,7 @@ EXPORT_SYMBOL(ap_owned_by_def_drv); * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check * - * Note: the ap_perms_mutex must be locked by the caller of this function. + * Note: the ap_attr_mutex must be locked by the caller of this function. * * Return: an int specifying whether each APQN is reserved for the host (1) or * not (0) @@ -895,12 +963,10 @@ int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm, int card, queue, rc = 0; for (card = 0; !rc && card < AP_DEVICES; card++) - if (test_bit_inv(card, apm) && - test_bit_inv(card, ap_perms.apm)) + if (test_bit_inv(card, apm)) for (queue = 0; !rc && queue < AP_DOMAINS; queue++) - if (test_bit_inv(queue, aqm) && - test_bit_inv(queue, ap_perms.aqm)) - rc = 1; + if (test_bit_inv(queue, aqm)) + rc = ap_owned_by_def_drv(card, queue); return rc; } @@ -924,15 +990,27 @@ static int ap_device_probe(struct device *dev) */ card = AP_QID_CARD(to_ap_queue(dev)->qid); queue = AP_QID_QUEUE(to_ap_queue(dev)->qid); - mutex_lock(&ap_perms_mutex); - devres = test_bit_inv(card, ap_perms.apm) && - test_bit_inv(queue, ap_perms.aqm); - mutex_unlock(&ap_perms_mutex); - drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT; - if (!!devres != !!drvres) - goto out; + if (ap_dev->driver_override) { + if (strcmp(ap_dev->driver_override, + ap_drv->driver.name)) + goto out; + } else { + mutex_lock(&ap_attr_mutex); + devres = test_bit_inv(card, ap_perms.apm) && + test_bit_inv(queue, ap_perms.aqm); + mutex_unlock(&ap_attr_mutex); + drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT; + if (!!devres != !!drvres) + goto out; + } } + /* + * Rearm the bindings complete completion to trigger + * bindings complete when all devices are bound again + */ + reinit_completion(&ap_apqn_bindings_complete); + /* Add queue/card to list of active queues/cards */ spin_lock_bh(&ap_queues_lock); if (is_queue_dev(dev)) @@ -947,13 +1025,20 @@ static int ap_device_probe(struct device *dev) if (is_queue_dev(dev)) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); - } else { - ap_check_bindings_complete(); } out: - if (rc) + if (rc) { put_device(dev); + } else { + if (is_queue_dev(dev)) { + pr_debug("queue=%02x.%04x new driver=%s\n", + card, queue, ap_drv->driver.name); + } else { + pr_debug("card=%02x new driver=%s\n", + to_ap_card(dev)->id, ap_drv->driver.name); + } + } return rc; } @@ -1006,11 +1091,16 @@ int ap_driver_register(struct ap_driver *ap_drv, struct module *owner, char *name) { struct device_driver *drv = &ap_drv->driver; + int rc; drv->bus = &ap_bus_type; drv->owner = owner; drv->name = name; - return driver_register(drv); + rc = driver_register(drv); + + ap_check_bindings_complete(); + + return rc; } EXPORT_SYMBOL(ap_driver_register); @@ -1020,34 +1110,85 @@ void ap_driver_unregister(struct ap_driver *ap_drv) } EXPORT_SYMBOL(ap_driver_unregister); -void ap_bus_force_rescan(void) +/* + * Enforce a synchronous AP bus rescan. + * Returns true if the bus scan finds a change in the AP configuration + * and AP devices have been added or deleted when this function returns. + */ +bool ap_bus_force_rescan(void) { - /* processing a asynchronous bus rescan */ - del_timer(&ap_config_timer); - queue_work(system_long_wq, &ap_scan_work); - flush_work(&ap_scan_work); + unsigned long scan_counter = atomic64_read(&ap_scan_bus_count); + bool rc = false; + + pr_debug("> scan counter=%lu\n", scan_counter); + + /* Only trigger AP bus scans after the initial scan is done */ + if (scan_counter <= 0) + goto out; + + /* + * There is one unlikely but nevertheless valid scenario where the + * thread holding the mutex may try to send some crypto load but + * all cards are offline so a rescan is triggered which causes + * a recursive call of ap_bus_force_rescan(). A simple return if + * the mutex is already locked by this thread solves this. + */ + if (mutex_is_locked(&ap_scan_bus_mutex)) { + if (ap_scan_bus_task == current) + goto out; + } + + /* Try to acquire the AP scan bus mutex */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + /* mutex acquired, run the AP bus scan */ + ap_scan_bus_task = current; + ap_scan_bus_result = ap_scan_bus(); + rc = ap_scan_bus_result; + ap_scan_bus_task = NULL; + mutex_unlock(&ap_scan_bus_mutex); + goto out; + } + + /* + * Mutex acquire failed. So there is currently another task + * already running the AP bus scan. Then let's simple wait + * for the lock which means the other task has finished and + * stored the result in ap_scan_bus_result. + */ + if (mutex_lock_interruptible(&ap_scan_bus_mutex)) { + /* some error occurred, ignore and go out */ + goto out; + } + rc = ap_scan_bus_result; + mutex_unlock(&ap_scan_bus_mutex); + +out: + pr_debug("rc=%d\n", rc); + return rc; } EXPORT_SYMBOL(ap_bus_force_rescan); /* * A config change has happened, force an ap bus rescan. */ -void ap_bus_cfg_chg(void) +static int ap_bus_cfg_chg(struct notifier_block *nb, + unsigned long action, void *data) { - AP_DBF_DBG("%s config change, forcing bus rescan\n", __func__); + if (action != CHSC_NOTIFY_AP_CFG) + return NOTIFY_DONE; + + pr_debug("config change, forcing bus rescan\n"); ap_bus_force_rescan(); + + return NOTIFY_OK; } -/* - * hex2bitmap() - parse hex mask string and set bitmap. - * Valid strings are "0x012345678" with at least one valid hex number. - * Rest of the bitmap to the right is padded with 0. No spaces allowed - * within the string, the leading 0x may be omitted. - * Returns the bitmask with exactly the bits set as given by the hex - * string (both in big endian order). - */ -static int hex2bitmap(const char *str, unsigned long *bitmap, int bits) +static struct notifier_block ap_bus_nb = { + .notifier_call = ap_bus_cfg_chg, +}; + +int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits) { int i, n, b; @@ -1074,6 +1215,7 @@ static int hex2bitmap(const char *str, unsigned long *bitmap, int bits) return -EINVAL; return 0; } +EXPORT_SYMBOL(ap_hex2bitmap); /* * modify_bitmap() - parse bitmask argument and modify an existing @@ -1094,7 +1236,7 @@ static int hex2bitmap(const char *str, unsigned long *bitmap, int bits) */ static int modify_bitmap(const char *str, unsigned long *bitmap, int bits) { - int a, i, z; + unsigned long a, i, z; char *np, sign; /* bits needs to be a multiple of 8 */ @@ -1139,7 +1281,7 @@ static int ap_parse_bitmap_str(const char *str, unsigned long *bitmap, int bits, rc = modify_bitmap(str, newmap, bits); } else { memset(newmap, 0, size); - rc = hex2bitmap(str, newmap, bits); + rc = ap_hex2bitmap(str, newmap, bits); } return rc; } @@ -1205,7 +1347,7 @@ static BUS_ATTR_RW(ap_domain); static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1219,7 +1361,7 @@ static BUS_ATTR_RO(ap_control_domain_mask); static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1233,7 +1375,7 @@ static BUS_ATTR_RO(ap_usage_domain_mask); static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1254,7 +1396,7 @@ static BUS_ATTR_RO(ap_interrupts); static ssize_t config_time_show(const struct bus_type *bus, char *buf) { - return sysfs_emit(buf, "%d\n", ap_config_time); + return sysfs_emit(buf, "%d\n", ap_scan_bus_time); } static ssize_t config_time_store(const struct bus_type *bus, @@ -1264,8 +1406,8 @@ static ssize_t config_time_store(const struct bus_type *bus, if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120) return -EINVAL; - ap_config_time = time; - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + ap_scan_bus_time = time; + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); return count; } @@ -1349,12 +1491,12 @@ static ssize_t apmask_show(const struct bus_type *bus, char *buf) { int rc; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n", ap_perms.apm[0], ap_perms.apm[1], ap_perms.apm[2], ap_perms.apm[3]); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -1364,6 +1506,7 @@ static int __verify_card_reservations(struct device_driver *drv, void *data) int rc = 0; struct ap_driver *ap_drv = to_ap_drv(drv); unsigned long *newapm = (unsigned long *)data; + unsigned long aqm_any[BITS_TO_LONGS(AP_DOMAINS)]; /* * increase the driver's module refcounter to be sure it is not @@ -1373,7 +1516,8 @@ static int __verify_card_reservations(struct device_driver *drv, void *data) return 0; if (ap_drv->in_use) { - rc = ap_drv->in_use(newapm, ap_perms.aqm); + bitmap_fill(aqm_any, AP_DOMAINS); + rc = ap_drv->in_use(newapm, aqm_any); if (rc) rc = -EBUSY; } @@ -1402,18 +1546,31 @@ static int apmask_commit(unsigned long *newapm) memcpy(ap_perms.apm, newapm, APMASKSIZE); + /* + * Update ap_apmask_aqmask_in_use. Note that the + * ap_attr_mutex has to be obtained here. + */ + ap_apmask_aqmask_in_use = + bitmap_full(ap_perms.apm, AP_DEVICES) && + bitmap_full(ap_perms.aqm, AP_DOMAINS) ? + false : true; + return 0; } static ssize_t apmask_store(const struct bus_type *bus, const char *buf, size_t count) { - int rc, changes = 0; DECLARE_BITMAP(newapm, AP_DEVICES); + int rc = -EINVAL, changes = 0; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; + /* Do not allow apmask/aqmask if driver override is active */ + if (ap_driver_override_ctr) + goto done; + rc = ap_parse_bitmap_str(buf, ap_perms.apm, AP_DEVICES, newapm); if (rc) goto done; @@ -1423,7 +1580,7 @@ static ssize_t apmask_store(const struct bus_type *bus, const char *buf, rc = apmask_commit(newapm); done: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (rc) return rc; @@ -1441,12 +1598,12 @@ static ssize_t aqmask_show(const struct bus_type *bus, char *buf) { int rc; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n", ap_perms.aqm[0], ap_perms.aqm[1], ap_perms.aqm[2], ap_perms.aqm[3]); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -1456,6 +1613,7 @@ static int __verify_queue_reservations(struct device_driver *drv, void *data) int rc = 0; struct ap_driver *ap_drv = to_ap_drv(drv); unsigned long *newaqm = (unsigned long *)data; + unsigned long apm_any[BITS_TO_LONGS(AP_DEVICES)]; /* * increase the driver's module refcounter to be sure it is not @@ -1465,7 +1623,8 @@ static int __verify_queue_reservations(struct device_driver *drv, void *data) return 0; if (ap_drv->in_use) { - rc = ap_drv->in_use(ap_perms.apm, newaqm); + bitmap_fill(apm_any, AP_DEVICES); + rc = ap_drv->in_use(apm_any, newaqm); if (rc) rc = -EBUSY; } @@ -1494,18 +1653,31 @@ static int aqmask_commit(unsigned long *newaqm) memcpy(ap_perms.aqm, newaqm, AQMASKSIZE); + /* + * Update ap_apmask_aqmask_in_use. Note that the + * ap_attr_mutex has to be obtained here. + */ + ap_apmask_aqmask_in_use = + bitmap_full(ap_perms.apm, AP_DEVICES) && + bitmap_full(ap_perms.aqm, AP_DOMAINS) ? + false : true; + return 0; } static ssize_t aqmask_store(const struct bus_type *bus, const char *buf, size_t count) { - int rc, changes = 0; DECLARE_BITMAP(newaqm, AP_DOMAINS); + int rc = -EINVAL, changes = 0; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; + /* Do not allow apmask/aqmask if driver override is active */ + if (ap_driver_override_ctr) + goto done; + rc = ap_parse_bitmap_str(buf, ap_perms.aqm, AP_DOMAINS, newaqm); if (rc) goto done; @@ -1515,7 +1687,7 @@ static ssize_t aqmask_store(const struct bus_type *bus, const char *buf, rc = aqmask_commit(newaqm); done: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (rc) return rc; @@ -1562,11 +1734,20 @@ static ssize_t bindings_show(const struct bus_type *bus, char *buf) static BUS_ATTR_RO(bindings); +static ssize_t bindings_complete_count_show(const struct bus_type *bus, + char *buf) +{ + return sysfs_emit(buf, "%llu\n", + atomic64_read(&ap_bindings_complete_count)); +} + +static BUS_ATTR_RO(bindings_complete_count); + static ssize_t features_show(const struct bus_type *bus, char *buf) { int n = 0; - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "-\n"); if (ap_qci_info->apsc) @@ -1602,12 +1783,13 @@ static struct attribute *ap_bus_attrs[] = { &bus_attr_aqmask.attr, &bus_attr_scans.attr, &bus_attr_bindings.attr, + &bus_attr_bindings_complete_count.attr, &bus_attr_features.attr, NULL, }; ATTRIBUTE_GROUPS(ap_bus); -static struct bus_type ap_bus_type = { +static const struct bus_type ap_bus_type = { .name = "ap", .bus_groups = ap_bus_groups, .match = &ap_bus_match, @@ -1798,12 +1980,12 @@ static inline void ap_scan_rm_card_dev_and_queue_devs(struct ap_card *ac) */ static inline void ap_scan_domains(struct ap_card *ac) { - int rc, dom, depth, type, ml; + struct ap_tapq_hwinfo hwinfo; bool decfg, chkstop; struct ap_queue *aq; struct device *dev; - unsigned int func; ap_qid_t qid; + int rc, dom; /* * Go through the configuration for the domains and compare them @@ -1826,8 +2008,7 @@ static inline void ap_scan_domains(struct ap_card *ac) goto put_dev_and_continue; } /* domain is valid, get info from this APQN */ - rc = ap_queue_info(qid, &type, &func, &depth, - &ml, &decfg, &chkstop); + rc = ap_queue_info(qid, &hwinfo, &decfg, &chkstop); switch (rc) { case -1: if (dev) { @@ -1843,15 +2024,15 @@ static inline void ap_scan_domains(struct ap_card *ac) } /* if no queue device exists, create a new one */ if (!aq) { - aq = ap_queue_create(qid, ac->ap_dev.device_type); + aq = ap_queue_create(qid, ac); if (!aq) { AP_DBF_WARN("%s(%d,%d) ap_queue_create() failed\n", __func__, ac->id, dom); continue; } - aq->card = ac; aq->config = !decfg; aq->chkstop = chkstop; + aq->se_bstate = hwinfo.bs; dev = &aq->ap_dev.device; dev->bus = &ap_bus_type; dev->parent = &ac->ap_dev.device; @@ -1865,19 +2046,24 @@ static inline void ap_scan_domains(struct ap_card *ac) } /* get it and thus adjust reference counter */ get_device(dev); - if (decfg) + if (decfg) { AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n", __func__, ac->id, dom); - else if (chkstop) + } else if (chkstop) { AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n", __func__, ac->id, dom); - else + } else { + /* nudge the queue's state machine */ + ap_queue_init_state(aq); AP_DBF_INFO("%s(%d,%d) new queue dev created\n", __func__, ac->id, dom); + } goto put_dev_and_continue; } /* handle state changes on already existing queue device */ spin_lock_bh(&aq->lock); + /* SE bind state */ + aq->se_bstate = hwinfo.bs; /* checkstop state */ if (chkstop && !aq->chkstop) { /* checkstop on */ @@ -1887,21 +2073,19 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop on\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev checkstop on\n", + ac->id, dom); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); goto put_dev_and_continue; } else if (!chkstop && aq->chkstop) { /* checkstop off */ aq->chkstop = false; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev checkstop off\n", + ac->id, dom); goto put_dev_and_continue; } /* config state change */ @@ -1913,8 +2097,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_DECONFIGURED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config off\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev config off\n", + ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); @@ -1922,13 +2106,11 @@ static inline void ap_scan_domains(struct ap_card *ac) } else if (!decfg && !aq->config) { /* config on this queue device */ aq->config = true; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config on\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev config on\n", + ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); goto put_dev_and_continue; } @@ -1955,11 +2137,11 @@ put_dev_and_continue: */ static inline void ap_scan_adapter(int ap) { - int rc, dom, depth, type, comp_type, ml; + struct ap_tapq_hwinfo hwinfo; + int rc, dom, comp_type; bool decfg, chkstop; struct ap_card *ac; struct device *dev; - unsigned int func; ap_qid_t qid; /* Is there currently a card device for this adapter ? */ @@ -1989,8 +2171,7 @@ static inline void ap_scan_adapter(int ap) for (dom = 0; dom <= ap_max_domain_id; dom++) if (ap_test_config_usage_domain(dom)) { qid = AP_MKQID(ap, dom); - if (ap_queue_info(qid, &type, &func, &depth, - &ml, &decfg, &chkstop) > 0) + if (ap_queue_info(qid, &hwinfo, &decfg, &chkstop) > 0) break; } if (dom > ap_max_domain_id) { @@ -2001,12 +2182,12 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no type info (no APQN found), ignored\n", - __func__, ap); + pr_debug("(%d) no type info (no APQN found), ignored\n", + ap); } return; } - if (!type) { + if (!hwinfo.at) { /* No apdater type info available, an unusable adapter */ if (ac) { AP_DBF_INFO("%s(%d) no valid type (0) info, rm card and queue devs\n", @@ -2014,23 +2195,22 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no valid type (0) info, ignored\n", - __func__, ap); + pr_debug("(%d) no valid type (0) info, ignored\n", ap); } return; } + hwinfo.value &= TAPQ_CARD_HWINFO_MASK; /* filter card specific hwinfo */ if (ac) { /* Check APQN against existing card device for changes */ - if (ac->raw_hwtype != type) { + if (ac->hwinfo.at != hwinfo.at) { AP_DBF_INFO("%s(%d) hwtype %d changed, rm card and queue devs\n", - __func__, ap, type); + __func__, ap, hwinfo.at); ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); ac = NULL; - } else if ((ac->functions & TAPQ_CARD_FUNC_CMP_MASK) != - (func & TAPQ_CARD_FUNC_CMP_MASK)) { + } else if (ac->hwinfo.fac != hwinfo.fac) { AP_DBF_INFO("%s(%d) functions 0x%08x changed, rm card and queue devs\n", - __func__, ap, func); + __func__, ap, hwinfo.fac); ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); ac = NULL; @@ -2064,13 +2244,13 @@ static inline void ap_scan_adapter(int ap) if (!ac) { /* Build a new card device */ - comp_type = ap_get_compatible_type(qid, type, func); + comp_type = ap_get_compatible_type(qid, hwinfo.at, hwinfo.fac); if (!comp_type) { AP_DBF_WARN("%s(%d) type %d, can't get compatibility type\n", - __func__, ap, type); + __func__, ap, hwinfo.at); return; } - ac = ap_card_create(ap, depth, type, comp_type, func, ml); + ac = ap_card_create(ap, hwinfo, comp_type); if (!ac) { AP_DBF_WARN("%s(%d) ap_card_create() failed\n", __func__, ap); @@ -2101,13 +2281,13 @@ static inline void ap_scan_adapter(int ap) get_device(dev); if (decfg) AP_DBF_INFO("%s(%d) new (decfg) card dev type=%d func=0x%08x created\n", - __func__, ap, type, func); + __func__, ap, hwinfo.at, hwinfo.fac); else if (chkstop) AP_DBF_INFO("%s(%d) new (chkstop) card dev type=%d func=0x%08x created\n", - __func__, ap, type, func); + __func__, ap, hwinfo.at, hwinfo.fac); else AP_DBF_INFO("%s(%d) new card dev type=%d func=0x%08x created\n", - __func__, ap, type, func); + __func__, ap, hwinfo.at, hwinfo.fac); } /* Verify the domains and the queue devices for this card */ @@ -2129,33 +2309,90 @@ static inline void ap_scan_adapter(int ap) */ static bool ap_get_configuration(void) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return false; memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); - ap_fetch_qci_info(ap_qci_info); + ap_qci(ap_qci_info); return memcmp(ap_qci_info, ap_qci_info_old, sizeof(struct ap_config_info)) != 0; } +/* + * ap_config_has_new_aps - Check current against old qci info if + * new adapters have appeared. Returns true if at least one new + * adapter in the apm mask is showing up. Existing adapters or + * receding adapters are not counted. + */ +static bool ap_config_has_new_aps(void) +{ + + unsigned long m[BITS_TO_LONGS(AP_DEVICES)]; + + if (!ap_qci_info->flags) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->apm, + (unsigned long *)ap_qci_info_old->apm, AP_DEVICES); + if (!bitmap_empty(m, AP_DEVICES)) + return true; + + return false; +} + +/* + * ap_config_has_new_doms - Check current against old qci info if + * new (usage) domains have appeared. Returns true if at least one + * new domain in the aqm mask is showing up. Existing domains or + * receding domains are not counted. + */ +static bool ap_config_has_new_doms(void) +{ + unsigned long m[BITS_TO_LONGS(AP_DOMAINS)]; + + if (!ap_qci_info->flags) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm, + (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS); + if (!bitmap_empty(m, AP_DOMAINS)) + return true; + + return false; +} + /** * ap_scan_bus(): Scan the AP bus for new devices - * Runs periodically, workqueue timer (ap_config_time) - * @unused: Unused pointer. + * Always run under mutex ap_scan_bus_mutex protection + * which needs to get locked/unlocked by the caller! + * Returns true if any config change has been detected + * during the scan, otherwise false. */ -static void ap_scan_bus(struct work_struct *unused) +static bool ap_scan_bus(void) { - int ap, config_changed = 0; + bool config_changed; + int ap; + + pr_debug(">\n"); - /* config change notify */ + /* (re-)fetch configuration via QCI */ config_changed = ap_get_configuration(); - if (config_changed) + if (config_changed) { + if (ap_config_has_new_aps() || ap_config_has_new_doms()) { + /* + * Appearance of new adapters and/or domains need to + * build new ap devices which need to get bound to an + * device driver. Thus reset the APQN bindings complete + * completion. + */ + reinit_completion(&ap_apqn_bindings_complete); + } + /* post a config change notify */ notify_config_changed(); + } ap_select_domain(); - AP_DBF_DBG("%s running\n", __func__); - /* loop over all possible adapters */ for (ap = 0; ap <= ap_max_adapter_id; ap++) ap_scan_adapter(ap); @@ -2178,23 +2415,132 @@ static void ap_scan_bus(struct work_struct *unused) } if (atomic64_inc_return(&ap_scan_bus_count) == 1) { - AP_DBF_DBG("%s init scan complete\n", __func__); + pr_debug("init scan complete\n"); ap_send_init_scan_done_uevent(); - ap_check_bindings_complete(); } - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + ap_check_bindings_complete(); + + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); + + pr_debug("< config_changed=%d\n", config_changed); + + return config_changed; +} + +/* + * Callback for the ap_scan_bus_timer + * Runs periodically, workqueue timer (ap_scan_bus_time) + */ +static void ap_scan_bus_timer_callback(struct timer_list *unused) +{ + /* + * schedule work into the system long wq which when + * the work is finally executed, calls the AP bus scan. + */ + queue_work(system_long_wq, &ap_scan_bus_work); +} + +/* + * Callback for the ap_scan_bus_work + */ +static void ap_scan_bus_wq_callback(struct work_struct *unused) +{ + /* + * Try to invoke an ap_scan_bus(). If the mutex acquisition + * fails there is currently another task already running the + * AP scan bus and there is no need to wait and re-trigger the + * scan again. Please note at the end of the scan bus function + * the AP scan bus timer is re-armed which triggers then the + * ap_scan_bus_timer_callback which enqueues a work into the + * system_long_wq which invokes this function here again. + */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + ap_scan_bus_task = current; + ap_scan_bus_result = ap_scan_bus(); + ap_scan_bus_task = NULL; + mutex_unlock(&ap_scan_bus_mutex); + } +} + +static inline void __exit ap_async_exit(void) +{ + if (ap_thread_flag) + ap_poll_thread_stop(); + chsc_notifier_unregister(&ap_bus_nb); + cancel_work(&ap_scan_bus_work); + hrtimer_cancel(&ap_poll_timer); + timer_delete(&ap_scan_bus_timer); +} + +static inline int __init ap_async_init(void) +{ + int rc; + + /* Setup the AP bus rescan timer. */ + timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0); + + /* + * Setup the high resolution poll timer. + * If we are running under z/VM adjust polling to z/VM polling rate. + */ + if (machine_is_vm()) + poll_high_timeout = 1500000; + hrtimer_setup(&ap_poll_timer, ap_poll_timeout, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + + queue_work(system_long_wq, &ap_scan_bus_work); + + rc = chsc_notifier_register(&ap_bus_nb); + if (rc) + goto out; + + /* Start the low priority AP bus poll thread. */ + if (!ap_thread_flag) + return 0; + + rc = ap_poll_thread_start(); + if (rc) + goto out_notifier; + + return 0; + +out_notifier: + chsc_notifier_unregister(&ap_bus_nb); +out: + cancel_work(&ap_scan_bus_work); + hrtimer_cancel(&ap_poll_timer); + timer_delete(&ap_scan_bus_timer); + return rc; +} + +static inline void ap_irq_exit(void) +{ + if (ap_irq_flag) + unregister_adapter_interrupt(&ap_airq); } -static void ap_config_timeout(struct timer_list *unused) +static inline int __init ap_irq_init(void) { - queue_work(system_long_wq, &ap_scan_work); + int rc; + + if (!ap_interrupts_available() || !ap_useirq) + return 0; + + rc = register_adapter_interrupt(&ap_airq); + ap_irq_flag = (rc == 0); + + return rc; } -static int __init ap_debug_init(void) +static inline void ap_debug_exit(void) +{ + debug_unregister(ap_dbf_info); +} + +static inline int __init ap_debug_init(void) { ap_dbf_info = debug_register("ap", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + AP_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(ap_dbf_info, &debug_sprintf_view); debug_set_level(ap_dbf_info, DBF_ERR); @@ -2212,14 +2558,14 @@ static void __init ap_perms_init(void) if (apm_str) { memset(&ap_perms.apm, 0, sizeof(ap_perms.apm)); ap_parse_mask_str(apm_str, ap_perms.apm, AP_DEVICES, - &ap_perms_mutex); + &ap_attr_mutex); } /* aqm kernel parameter string */ if (aqm_str) { memset(&ap_perms.aqm, 0, sizeof(ap_perms.aqm)); ap_parse_mask_str(aqm_str, ap_perms.aqm, AP_DOMAINS, - &ap_perms_mutex); + &ap_attr_mutex); } } @@ -2232,18 +2578,26 @@ static int __init ap_module_init(void) { int rc; - rc = ap_debug_init(); - if (rc) - return rc; - if (!ap_instructions_available()) { pr_warn("The hardware system does not support AP instructions\n"); return -ENODEV; } + rc = ap_debug_init(); + if (rc) + return rc; + /* init ap_queue hashtable */ hash_init(ap_queues); + /* create ap msg buffer memory pool */ + ap_msg_pool = mempool_create_kmalloc_pool(ap_msg_pool_min_items, + AP_DEFAULT_MAX_MSG_SIZE); + if (!ap_msg_pool) { + rc = -ENOMEM; + goto out; + } + /* set up the AP permissions (ioctls, ap and aq masks) */ ap_perms_init(); @@ -2259,12 +2613,6 @@ static int __init ap_module_init(void) ap_domain_index = -1; } - /* enable interrupts if available */ - if (ap_interrupts_available() && ap_useirq) { - rc = register_adapter_interrupt(&ap_airq); - ap_irq_flag = (rc == 0); - } - /* Create /sys/bus/ap. */ rc = bus_register(&ap_bus_type); if (rc) @@ -2277,38 +2625,39 @@ static int __init ap_module_init(void) goto out_bus; ap_root_device->bus = &ap_bus_type; - /* Setup the AP bus rescan timer. */ - timer_setup(&ap_config_timer, ap_config_timeout, 0); - - /* - * Setup the high resolution poll timer. - * If we are running under z/VM adjust polling to z/VM polling rate. - */ - if (MACHINE_IS_VM) - poll_high_timeout = 1500000; - hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - ap_poll_timer.function = ap_poll_timeout; - - /* Start the low priority AP bus poll thread. */ - if (ap_thread_flag) { - rc = ap_poll_thread_start(); - if (rc) - goto out_work; - } + /* enable interrupts if available */ + rc = ap_irq_init(); + if (rc) + goto out_device; - queue_work(system_long_wq, &ap_scan_work); + /* Setup asynchronous work (timers, workqueue, etc). */ + rc = ap_async_init(); + if (rc) + goto out_irq; return 0; -out_work: - hrtimer_cancel(&ap_poll_timer); +out_irq: + ap_irq_exit(); +out_device: root_device_unregister(ap_root_device); out_bus: bus_unregister(&ap_bus_type); out: - if (ap_irq_flag) - unregister_adapter_interrupt(&ap_airq); - kfree(ap_qci_info); + mempool_destroy(ap_msg_pool); + ap_debug_exit(); return rc; } -device_initcall(ap_module_init); + +static void __exit ap_module_exit(void) +{ + ap_async_exit(); + ap_irq_exit(); + root_device_unregister(ap_root_device); + bus_unregister(&ap_bus_type); + mempool_destroy(ap_msg_pool); + ap_debug_exit(); +} + +module_init(ap_module_init); +module_exit(ap_module_exit); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index be54b070c031..51e08f27bd75 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -76,16 +76,6 @@ static inline int ap_test_bit(unsigned int *ptr, unsigned int nr) #define AP_DEVICE_TYPE_CEX8 14 /* - * Known function facilities - */ -#define AP_FUNC_MEX4K 1 -#define AP_FUNC_CRT4K 2 -#define AP_FUNC_COPRO 3 -#define AP_FUNC_ACCEL 4 -#define AP_FUNC_EP11 5 -#define AP_FUNC_APXA 6 - -/* * AP queue state machine states */ enum ap_sm_state { @@ -168,7 +158,7 @@ struct ap_driver { struct ap_config_info *old_config_info); }; -#define to_ap_drv(x) container_of((x), struct ap_driver, driver) +#define to_ap_drv(x) container_of_const((x), struct ap_driver, driver) int ap_driver_register(struct ap_driver *, struct module *, char *); void ap_driver_unregister(struct ap_driver *); @@ -176,15 +166,14 @@ void ap_driver_unregister(struct ap_driver *); struct ap_device { struct device device; int device_type; /* AP device type. */ + const char *driver_override; }; #define to_ap_dev(x) container_of((x), struct ap_device, device) struct ap_card { struct ap_device ap_dev; - int raw_hwtype; /* AP raw hardware type. */ - unsigned int functions; /* TAPQ GR2 upper 32 facility bits */ - int queue_depth; /* AP queue depth.*/ + struct ap_tapq_hwinfo hwinfo; /* TAPQ GR2 content */ int id; /* AP card number. */ unsigned int maxmsgsize; /* AP msg limit for this card */ bool config; /* configured state */ @@ -192,7 +181,7 @@ struct ap_card { atomic64_t total_request_count; /* # requests ever for this AP device.*/ }; -#define TAPQ_CARD_FUNC_CMP_MASK 0xFFFF0000 +#define TAPQ_CARD_HWINFO_MASK 0xFFFF0000FFFF0F0FUL #define ASSOC_IDX_INVALID 0x10000 #define to_ap_card(x) container_of((x), struct ap_card, ap_dev.device) @@ -206,7 +195,7 @@ struct ap_queue { bool config; /* configured state */ bool chkstop; /* checkstop state */ ap_qid_t qid; /* AP queue id. */ - bool interrupt; /* indicate if interrupts are enabled */ + unsigned int se_bstate; /* SE bind state (BS) */ unsigned int assoc_idx; /* SE association index */ int queue_count; /* # messages currently on AP queue. */ int pendingq_count; /* # requests on pendingq list. */ @@ -226,6 +215,11 @@ struct ap_queue { typedef enum ap_sm_wait (ap_func_t)(struct ap_queue *queue); +struct ap_response_type { + struct completion work; + int type; +}; + struct ap_message { struct list_head list; /* Request queueing. */ unsigned long psmid; /* Message id. */ @@ -234,7 +228,7 @@ struct ap_message { size_t bufsize; /* allocated msg buffer size */ u16 flags; /* Flags, see AP_MSG_FLAG_xxx */ int rc; /* Return code for this message */ - void *private; /* ap driver private pointer. */ + struct ap_response_type response; /* receive is called from tasklet context */ void (*receive)(struct ap_queue *, struct ap_message *, struct ap_message *); @@ -243,27 +237,10 @@ struct ap_message { #define AP_MSG_FLAG_SPECIAL 0x0001 /* flag msg as 'special' with NQAP */ #define AP_MSG_FLAG_USAGE 0x0002 /* CCA, EP11: usage (no admin) msg */ #define AP_MSG_FLAG_ADMIN 0x0004 /* CCA, EP11: admin (=control) msg */ +#define AP_MSG_FLAG_MEMPOOL 0x0008 /* ap msg buffer allocated via mempool */ -/** - * ap_init_message() - Initialize ap_message. - * Initialize a message before using. Otherwise this might result in - * unexpected behaviour. - */ -static inline void ap_init_message(struct ap_message *ap_msg) -{ - memset(ap_msg, 0, sizeof(*ap_msg)); -} - -/** - * ap_release_message() - Release ap_message. - * Releases all memory used internal within the ap_message struct - * Currently this is the message and private field. - */ -static inline void ap_release_message(struct ap_message *ap_msg) -{ - kfree_sensitive(ap_msg->msg); - kfree_sensitive(ap_msg->private); -} +int ap_init_apmsg(struct ap_message *ap_msg, u32 flags); +void ap_release_apmsg(struct ap_message *ap_msg); enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event); enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event); @@ -271,25 +248,27 @@ enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event); int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_flush_queue(struct ap_queue *aq); +bool ap_queue_usable(struct ap_queue *aq); void *ap_airq_ptr(void); int ap_sb_available(void); bool ap_is_se_guest(void); void ap_wait(enum ap_sm_wait wait); void ap_request_timeout(struct timer_list *t); -void ap_bus_force_rescan(void); +bool ap_bus_force_rescan(void); int ap_test_config_usage_domain(unsigned int domain); int ap_test_config_ctrl_domain(unsigned int domain); void ap_queue_init_reply(struct ap_queue *aq, struct ap_message *ap_msg); -struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type); +struct ap_queue *ap_queue_create(ap_qid_t qid, struct ap_card *ac); void ap_queue_prepare_remove(struct ap_queue *aq); void ap_queue_remove(struct ap_queue *aq); void ap_queue_init_state(struct ap_queue *aq); +void _ap_queue_init_state(struct ap_queue *aq); -struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, - int comp_type, unsigned int functions, int ml); +struct ap_card *ap_card_create(int id, struct ap_tapq_hwinfo info, + int comp_type); #define APMASKSIZE (BITS_TO_LONGS(AP_DEVICES) * sizeof(unsigned long)) #define AQMASKSIZE (BITS_TO_LONGS(AP_DOMAINS) * sizeof(unsigned long)) @@ -302,7 +281,9 @@ struct ap_perms { }; extern struct ap_perms ap_perms; -extern struct mutex ap_perms_mutex; +extern bool ap_apmask_aqmask_in_use; +extern int ap_driver_override_ctr; +extern struct mutex ap_attr_mutex; /* * Get ap_queue device for this qid. @@ -354,6 +335,28 @@ int ap_parse_mask_str(const char *str, struct mutex *lock); /* + * ap_hex2bitmap() - Convert a string containing a hexadecimal number (str) + * into a bitmap (bitmap) with bits set that correspond to the bits represented + * by the hex string. Input and output data is in big endian order. + * + * str - Input hex string of format "0x1234abcd". The leading "0x" is optional. + * At least one digit is required. Must be large enough to hold the number of + * bits represented by the bits parameter. + * + * bitmap - Pointer to a bitmap. Upon successful completion of this function, + * this bitmap will have bits set to match the value of str. If bitmap is longer + * than str, then the rightmost bits of bitmap are padded with zeros. Must be + * large enough to hold the number of bits represented by the bits parameter. + * + * bits - Length, in bits, of the bitmap represented by str. Must be a multiple + * of 8. + * + * Returns: 0 On success + * -EINVAL If str format is invalid or bits is not a multiple of 8. + */ +int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits); + +/* * Interface to wait for the AP bus to have done one initial ap bus * scan and all detected APQNs have been bound to device drivers. * If these both conditions are not fulfilled, this function blocks @@ -362,8 +365,12 @@ int ap_parse_mask_str(const char *str, * the return value is 0. If the timeout (in jiffies) hits instead * -ETIME is returned. On failures negative return values are * returned to the caller. + * It may be that the AP bus scan finds new devices. Then the + * condition that all APQNs are bound to their device drivers + * is reset to false and this call again blocks until either all + * APQNs are bound to a device driver or the timeout hits again. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout); +int ap_wait_apqn_bindings_complete(unsigned long timeout); void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg); void ap_send_online_uevent(struct ap_device *ap_dev, int online); diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c index b2bd477659a7..8102c8134c49 100644 --- a/drivers/s390/crypto/ap_card.c +++ b/drivers/s390/crypto/ap_card.c @@ -6,8 +6,7 @@ * Adjunct processor bus, card related code. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt #include <linux/init.h> #include <linux/slab.h> @@ -34,7 +33,7 @@ static ssize_t raw_hwtype_show(struct device *dev, { struct ap_card *ac = to_ap_card(dev); - return sysfs_emit(buf, "%d\n", ac->raw_hwtype); + return sysfs_emit(buf, "%d\n", ac->hwinfo.at); } static DEVICE_ATTR_RO(raw_hwtype); @@ -44,7 +43,7 @@ static ssize_t depth_show(struct device *dev, struct device_attribute *attr, { struct ap_card *ac = to_ap_card(dev); - return sysfs_emit(buf, "%d\n", ac->queue_depth); + return sysfs_emit(buf, "%d\n", ac->hwinfo.qd); } static DEVICE_ATTR_RO(depth); @@ -54,7 +53,7 @@ static ssize_t ap_functions_show(struct device *dev, { struct ap_card *ac = to_ap_card(dev); - return sysfs_emit(buf, "0x%08X\n", ac->functions); + return sysfs_emit(buf, "0x%08X\n", ac->hwinfo.fac); } static DEVICE_ATTR_RO(ap_functions); @@ -229,8 +228,8 @@ static void ap_card_device_release(struct device *dev) kfree(ac); } -struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, - int comp_type, unsigned int functions, int ml) +struct ap_card *ap_card_create(int id, struct ap_tapq_hwinfo hwinfo, + int comp_type) { struct ap_card *ac; @@ -240,12 +239,10 @@ struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, ac->ap_dev.device.release = ap_card_device_release; ac->ap_dev.device.type = &ap_card_type; ac->ap_dev.device_type = comp_type; - ac->raw_hwtype = raw_type; - ac->queue_depth = queue_depth; - ac->functions = functions; + ac->hwinfo = hwinfo; ac->id = id; - ac->maxmsgsize = ml > 0 ? - ml * AP_TAPQ_ML_FIELD_CHUNK_SIZE : AP_DEFAULT_MAX_MSG_SIZE; + ac->maxmsgsize = hwinfo.ml > 0 ? + hwinfo.ml * AP_TAPQ_ML_FIELD_CHUNK_SIZE : AP_DEFAULT_MAX_MSG_SIZE; return ac; } diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h index c083ce88a9a6..2f66271b8564 100644 --- a/drivers/s390/crypto/ap_debug.h +++ b/drivers/s390/crypto/ap_debug.h @@ -16,7 +16,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define AP_DBF_MAX_SPRINTF_ARGS 6 #define AP_DBF(...) \ debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__) @@ -26,8 +26,6 @@ debug_sprintf_event(ap_dbf_info, DBF_WARN, ##__VA_ARGS__) #define AP_DBF_INFO(...) \ debug_sprintf_event(ap_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define AP_DBF_DBG(...) \ - debug_sprintf_event(ap_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *ap_dbf_info; diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 1336e632adc4..4a32c1e19a1e 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -6,31 +6,46 @@ * Adjunct processor bus, queue related code. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt +#include <linux/export.h> #include <linux/init.h> #include <linux/slab.h> #include <asm/facility.h> +#define CREATE_TRACE_POINTS +#include <asm/trace/ap.h> + #include "ap_bus.h" #include "ap_debug.h" +EXPORT_TRACEPOINT_SYMBOL(s390_ap_nqap); +EXPORT_TRACEPOINT_SYMBOL(s390_ap_dqap); + static void __ap_flush_queue(struct ap_queue *aq); /* * some AP queue helper functions */ +static inline bool ap_q_supported_in_se(struct ap_queue *aq) +{ + return aq->card->hwinfo.ep11 || aq->card->hwinfo.accel; +} + static inline bool ap_q_supports_bind(struct ap_queue *aq) { - return ap_test_bit(&aq->card->functions, AP_FUNC_EP11) || - ap_test_bit(&aq->card->functions, AP_FUNC_ACCEL); + return aq->card->hwinfo.ep11 || aq->card->hwinfo.accel; } static inline bool ap_q_supports_assoc(struct ap_queue *aq) { - return ap_test_bit(&aq->card->functions, AP_FUNC_EP11); + return aq->card->hwinfo.ep11; +} + +static inline bool ap_q_needs_bind(struct ap_queue *aq) +{ + return ap_q_supports_bind(aq) && ap_sb_available(); } /** @@ -88,9 +103,17 @@ static inline struct ap_queue_status __ap_send(ap_qid_t qid, unsigned long psmid, void *msg, size_t msglen, int special) { + struct ap_queue_status status; + if (special) qid |= 0x400000UL; - return ap_nqap(qid, psmid, msg, msglen); + + status = ap_nqap(qid, psmid, msg, msglen); + + trace_s390_ap_nqap(AP_QID_CARD(qid), AP_QID_QUEUE(qid), + status.value, psmid); + + return status; } /* State machine definitions and helpers */ @@ -130,8 +153,13 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) parts++; } while (status.response_code == 0xFF && resgr0 != 0); + trace_s390_ap_dqap(AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + status.value, aq->reply->psmid); + switch (status.response_code) { case AP_RESPONSE_NORMAL: + print_hex_dump_debug("aprpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + aq->reply->msg, aq->reply->len, false); aq->queue_count = max_t(int, 0, aq->queue_count - 1); if (!status.queue_empty && !aq->queue_count) aq->queue_count++; @@ -165,6 +193,9 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) aq->queue_count = 0; list_splice_init(&aq->pendingq, &aq->requestq); aq->requestq_count += aq->pendingq_count; + pr_debug("queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n", + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + aq->pendingq_count, aq->requestq_count); aq->pendingq_count = 0; break; default: @@ -195,13 +226,13 @@ static enum ap_sm_wait ap_sm_read(struct ap_queue *aq) return AP_SM_WAIT_AGAIN; } aq->sm_state = AP_SM_STATE_IDLE; - return AP_SM_WAIT_NONE; + break; case AP_RESPONSE_NO_PENDING_REPLY: if (aq->queue_count > 0) - return aq->interrupt ? + return status.irq_enabled ? AP_SM_WAIT_INTERRUPT : AP_SM_WAIT_HIGH_TIMEOUT; aq->sm_state = AP_SM_STATE_IDLE; - return AP_SM_WAIT_NONE; + break; default: aq->dev_state = AP_DEV_STATE_ERROR; aq->last_err_rc = status.response_code; @@ -210,6 +241,16 @@ static enum ap_sm_wait ap_sm_read(struct ap_queue *aq) AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } + /* Check and maybe enable irq support (again) on this queue */ + if (!status.irq_enabled && status.queue_empty) { + void *lsi_ptr = ap_airq_ptr(); + + if (lsi_ptr && ap_queue_enable_irq(aq, lsi_ptr) == 0) { + aq->sm_state = AP_SM_STATE_SETIRQ_WAIT; + return AP_SM_WAIT_AGAIN; + } + } + return AP_SM_WAIT_NONE; } /** @@ -229,6 +270,8 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) /* Start the next request on the queue. */ ap_msg = list_entry(aq->requestq.next, struct ap_message, list); + print_hex_dump_debug("apreq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg->msg, ap_msg->len, false); status = __ap_send(qid, ap_msg->psmid, ap_msg->msg, ap_msg->len, ap_msg->flags & AP_MSG_FLAG_SPECIAL); @@ -242,14 +285,14 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) list_move_tail(&ap_msg->list, &aq->pendingq); aq->requestq_count--; aq->pendingq_count++; - if (aq->queue_count < aq->card->queue_depth) { + if (aq->queue_count < aq->card->hwinfo.qd) { aq->sm_state = AP_SM_STATE_WORKING; return AP_SM_WAIT_AGAIN; } fallthrough; case AP_RESPONSE_Q_FULL: aq->sm_state = AP_SM_STATE_QUEUE_FULL; - return aq->interrupt ? + return status.irq_enabled ? AP_SM_WAIT_INTERRUPT : AP_SM_WAIT_HIGH_TIMEOUT; case AP_RESPONSE_RESET_IN_PROGRESS: aq->sm_state = AP_SM_STATE_RESET_WAIT; @@ -302,7 +345,6 @@ static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq) case AP_RESPONSE_NORMAL: case AP_RESPONSE_RESET_IN_PROGRESS: aq->sm_state = AP_SM_STATE_RESET_WAIT; - aq->interrupt = false; aq->rapq_fbit = 0; return AP_SM_WAIT_LOW_TIMEOUT; default: @@ -324,17 +366,15 @@ static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq) static enum ap_sm_wait ap_sm_reset_wait(struct ap_queue *aq) { struct ap_queue_status status; + struct ap_tapq_hwinfo hwinfo; void *lsi_ptr; - if (aq->queue_count > 0 && aq->reply) - /* Try to read a completed message and get the status */ - status = ap_sm_recv(aq); - else - /* Get the status with TAPQ */ - status = ap_tapq(aq->qid, NULL); + /* Get the status with TAPQ */ + status = ap_test_queue(aq->qid, 1, &hwinfo); switch (status.response_code) { case AP_RESPONSE_NORMAL: + aq->se_bstate = hwinfo.bs; lsi_ptr = ap_airq_ptr(); if (lsi_ptr && ap_queue_enable_irq(aq, lsi_ptr) == 0) aq->sm_state = AP_SM_STATE_SETIRQ_WAIT; @@ -377,7 +417,6 @@ static enum ap_sm_wait ap_sm_setirq_wait(struct ap_queue *aq) if (status.irq_enabled == 1) { /* Irqs are now enabled */ - aq->interrupt = true; aq->sm_state = (aq->queue_count > 0) ? AP_SM_STATE_WORKING : AP_SM_STATE_IDLE; } @@ -407,9 +446,9 @@ static enum ap_sm_wait ap_sm_setirq_wait(struct ap_queue *aq) static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) { struct ap_queue_status status; - struct ap_tapq_gr2 info; + struct ap_tapq_hwinfo hwinfo; - status = ap_test_queue(aq->qid, 1, &info); + status = ap_test_queue(aq->qid, 1, &hwinfo); /* handle asynchronous error on this queue */ if (status.async && status.response_code) { aq->dev_state = AP_DEV_STATE_ERROR; @@ -428,14 +467,17 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) return AP_SM_WAIT_NONE; } + /* update queue's SE bind state */ + aq->se_bstate = hwinfo.bs; + /* check bs bits */ - switch (info.bs) { + switch (hwinfo.bs) { case AP_BS_Q_USABLE: /* association is through */ aq->sm_state = AP_SM_STATE_IDLE; - AP_DBF_DBG("%s queue 0x%02x.%04x associated with %u\n", - __func__, AP_QID_CARD(aq->qid), - AP_QID_QUEUE(aq->qid), aq->assoc_idx); + pr_debug("queue 0x%02x.%04x associated with %u\n", + AP_QID_CARD(aq->qid), + AP_QID_QUEUE(aq->qid), aq->assoc_idx); return AP_SM_WAIT_NONE; case AP_BS_Q_USABLE_NO_SECURE_KEY: /* association still pending */ @@ -446,7 +488,7 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) aq->dev_state = AP_DEV_STATE_ERROR; aq->last_err_rc = status.response_code; AP_DBF_WARN("%s bs 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", - __func__, info.bs, + __func__, hwinfo.bs, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } @@ -620,16 +662,21 @@ static ssize_t interrupt_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ap_queue *aq = to_ap_queue(dev); + struct ap_queue_status status; int rc = 0; spin_lock_bh(&aq->lock); - if (aq->sm_state == AP_SM_STATE_SETIRQ_WAIT) + if (aq->sm_state == AP_SM_STATE_SETIRQ_WAIT) { rc = sysfs_emit(buf, "Enable Interrupt pending.\n"); - else if (aq->interrupt) - rc = sysfs_emit(buf, "Interrupts enabled.\n"); - else - rc = sysfs_emit(buf, "Interrupts disabled.\n"); + } else { + status = ap_tapq(aq->qid, NULL); + if (status.irq_enabled) + rc = sysfs_emit(buf, "Interrupts enabled.\n"); + else + rc = sysfs_emit(buf, "Interrupts disabled.\n"); + } spin_unlock_bh(&aq->lock); + return rc; } @@ -668,22 +715,74 @@ static ssize_t ap_functions_show(struct device *dev, { struct ap_queue *aq = to_ap_queue(dev); struct ap_queue_status status; - struct ap_tapq_gr2 info; + struct ap_tapq_hwinfo hwinfo; - status = ap_test_queue(aq->qid, 1, &info); + status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } - return sysfs_emit(buf, "0x%08X\n", info.fac); + return sysfs_emit(buf, "0x%08X\n", hwinfo.fac); } static DEVICE_ATTR_RO(ap_functions); -#ifdef CONFIG_ZCRYPT_DEBUG +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + int rc; + + device_lock(dev); + if (ap_dev->driver_override) + rc = sysfs_emit(buf, "%s\n", ap_dev->driver_override); + else + rc = sysfs_emit(buf, "\n"); + device_unlock(dev); + + return rc; +} + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + int rc = -EINVAL; + bool old_value; + + if (mutex_lock_interruptible(&ap_attr_mutex)) + return -ERESTARTSYS; + + /* Do not allow driver override if apmask/aqmask is in use */ + if (ap_apmask_aqmask_in_use) + goto out; + + old_value = ap_dev->driver_override ? true : false; + rc = driver_set_override(dev, &ap_dev->driver_override, buf, count); + if (rc) + goto out; + if (old_value && !ap_dev->driver_override) + --ap_driver_override_ctr; + else if (!old_value && ap_dev->driver_override) + ++ap_driver_override_ctr; + + rc = count; + +out: + mutex_unlock(&ap_attr_mutex); + return rc; +} + +static DEVICE_ATTR_RW(driver_override); + +#ifdef CONFIG_AP_DEBUG static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -795,7 +894,8 @@ static struct attribute *ap_queue_dev_attrs[] = { &dev_attr_config.attr, &dev_attr_chkstop.attr, &dev_attr_ap_functions.attr, -#ifdef CONFIG_ZCRYPT_DEBUG + &dev_attr_driver_override.attr, +#ifdef CONFIG_AP_DEBUG &dev_attr_states.attr, &dev_attr_last_err_rc.attr, #endif @@ -821,19 +921,25 @@ static ssize_t se_bind_show(struct device *dev, { struct ap_queue *aq = to_ap_queue(dev); struct ap_queue_status status; - struct ap_tapq_gr2 info; + struct ap_tapq_hwinfo hwinfo; if (!ap_q_supports_bind(aq)) return sysfs_emit(buf, "-\n"); - status = ap_test_queue(aq->qid, 1, &info); + status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } - switch (info.bs) { + + /* update queue's SE bind state */ + spin_lock_bh(&aq->lock); + aq->se_bstate = hwinfo.bs; + spin_unlock_bh(&aq->lock); + + switch (hwinfo.bs) { case AP_BS_Q_USABLE: case AP_BS_Q_USABLE_NO_SECURE_KEY: return sysfs_emit(buf, "bound\n"); @@ -848,6 +954,7 @@ static ssize_t se_bind_store(struct device *dev, { struct ap_queue *aq = to_ap_queue(dev); struct ap_queue_status status; + struct ap_tapq_hwinfo hwinfo; bool value; int rc; @@ -859,34 +966,80 @@ static ssize_t se_bind_store(struct device *dev, if (rc) return rc; - if (value) { - /* bind, do BAPQ */ - spin_lock_bh(&aq->lock); - if (aq->sm_state < AP_SM_STATE_IDLE) { - spin_unlock_bh(&aq->lock); - return -EBUSY; - } - status = ap_bapq(aq->qid); - spin_unlock_bh(&aq->lock); - if (status.response_code) { - AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), - AP_QID_QUEUE(aq->qid)); - return -EIO; - } - } else { - /* unbind, set F bit arg and trigger RAPQ */ + if (!value) { + /* Unbind. Set F bit arg and trigger RAPQ */ spin_lock_bh(&aq->lock); __ap_flush_queue(aq); aq->rapq_fbit = 1; - aq->assoc_idx = ASSOC_IDX_INVALID; - aq->sm_state = AP_SM_STATE_RESET_START; - ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); - spin_unlock_bh(&aq->lock); + _ap_queue_init_state(aq); + rc = count; + goto out; } - return count; + /* Bind. Check current SE bind state */ + status = ap_test_queue(aq->qid, 1, &hwinfo); + if (status.response_code) { + AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + return -EIO; + } + + /* Update BS state */ + spin_lock_bh(&aq->lock); + aq->se_bstate = hwinfo.bs; + if (hwinfo.bs != AP_BS_Q_AVAIL_FOR_BINDING) { + AP_DBF_WARN("%s bind attempt with bs %d on queue 0x%02x.%04x\n", + __func__, hwinfo.bs, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = -EINVAL; + goto out; + } + + /* Check SM state */ + if (aq->sm_state < AP_SM_STATE_IDLE) { + rc = -EBUSY; + goto out; + } + + /* invoke BAPQ */ + status = ap_bapq(aq->qid); + if (status.response_code) { + AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = -EIO; + goto out; + } + aq->assoc_idx = ASSOC_IDX_INVALID; + + /* verify SE bind state */ + status = ap_test_queue(aq->qid, 1, &hwinfo); + if (status.response_code) { + AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = -EIO; + goto out; + } + aq->se_bstate = hwinfo.bs; + if (!(hwinfo.bs == AP_BS_Q_USABLE || + hwinfo.bs == AP_BS_Q_USABLE_NO_SECURE_KEY)) { + AP_DBF_WARN("%s BAPQ success, but bs shows %d on queue 0x%02x.%04x\n", + __func__, hwinfo.bs, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = -EIO; + goto out; + } + + /* SE bind was successful */ + AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = count; + +out: + spin_unlock_bh(&aq->lock); + return rc; } static DEVICE_ATTR_RW(se_bind); @@ -896,20 +1049,25 @@ static ssize_t se_associate_show(struct device *dev, { struct ap_queue *aq = to_ap_queue(dev); struct ap_queue_status status; - struct ap_tapq_gr2 info; + struct ap_tapq_hwinfo hwinfo; if (!ap_q_supports_assoc(aq)) return sysfs_emit(buf, "-\n"); - status = ap_test_queue(aq->qid, 1, &info); + status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } - switch (info.bs) { + /* update queue's SE bind state */ + spin_lock_bh(&aq->lock); + aq->se_bstate = hwinfo.bs; + spin_unlock_bh(&aq->lock); + + switch (hwinfo.bs) { case AP_BS_Q_USABLE: if (aq->assoc_idx == ASSOC_IDX_INVALID) { AP_DBF_WARN("%s AP_BS_Q_USABLE but invalid assoc_idx\n", __func__); @@ -931,6 +1089,7 @@ static ssize_t se_associate_store(struct device *dev, { struct ap_queue *aq = to_ap_queue(dev); struct ap_queue_status status; + struct ap_tapq_hwinfo hwinfo; unsigned int value; int rc; @@ -944,18 +1103,28 @@ static ssize_t se_associate_store(struct device *dev, if (value >= ASSOC_IDX_INVALID) return -EINVAL; + /* check current SE bind state */ + status = ap_test_queue(aq->qid, 1, &hwinfo); + if (status.response_code) { + AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + return -EIO; + } spin_lock_bh(&aq->lock); - - /* sm should be in idle state */ - if (aq->sm_state != AP_SM_STATE_IDLE) { - spin_unlock_bh(&aq->lock); - return -EBUSY; + aq->se_bstate = hwinfo.bs; + if (hwinfo.bs != AP_BS_Q_USABLE_NO_SECURE_KEY) { + AP_DBF_WARN("%s association attempt with bs %d on queue 0x%02x.%04x\n", + __func__, hwinfo.bs, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + rc = -EINVAL; + goto out; } - /* already associated or association pending ? */ - if (aq->assoc_idx != ASSOC_IDX_INVALID) { - spin_unlock_bh(&aq->lock); - return -EINVAL; + /* check SM state */ + if (aq->sm_state != AP_SM_STATE_IDLE) { + rc = -EBUSY; + goto out; } /* trigger the asynchronous association request */ @@ -966,17 +1135,20 @@ static ssize_t se_associate_store(struct device *dev, aq->sm_state = AP_SM_STATE_ASSOC_WAIT; aq->assoc_idx = value; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); - spin_unlock_bh(&aq->lock); break; default: - spin_unlock_bh(&aq->lock); AP_DBF_WARN("%s RC 0x%02x on aapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); - return -EIO; + rc = -EIO; + goto out; } - return count; + rc = count; + +out: + spin_unlock_bh(&aq->lock); + return rc; } static DEVICE_ATTR_RW(se_associate); @@ -1007,21 +1179,21 @@ static void ap_queue_device_release(struct device *dev) kfree(aq); } -struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type) +struct ap_queue *ap_queue_create(ap_qid_t qid, struct ap_card *ac) { struct ap_queue *aq; aq = kzalloc(sizeof(*aq), GFP_KERNEL); if (!aq) return NULL; + aq->card = ac; aq->ap_dev.device.release = ap_queue_device_release; aq->ap_dev.device.type = &ap_queue_type; - aq->ap_dev.device_type = device_type; - // add optional SE secure binding attributes group - if (ap_sb_available() && is_prot_virt_guest()) + aq->ap_dev.device_type = ac->ap_dev.device_type; + /* in SE environment add bind/associate attributes group */ + if (ap_is_se_guest() && ap_q_supported_in_se(aq)) aq->ap_dev.device.groups = ap_queue_dev_sb_attr_groups; aq->qid = qid; - aq->interrupt = false; spin_lock_init(&aq->lock); INIT_LIST_HEAD(&aq->pendingq); INIT_LIST_HEAD(&aq->requestq); @@ -1074,6 +1246,50 @@ int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg) EXPORT_SYMBOL(ap_queue_message); /** + * ap_queue_usable(): Check if queue is usable just now. + * @aq: The AP queue device to test for usability. + * This function is intended for the scheduler to query if it makes + * sense to enqueue a message into this AP queue device by calling + * ap_queue_message(). The perspective is very short-term as the + * state machine and device state(s) may change at any time. + */ +bool ap_queue_usable(struct ap_queue *aq) +{ + bool rc = true; + + spin_lock_bh(&aq->lock); + + /* check for not configured or checkstopped */ + if (!aq->config || aq->chkstop) { + rc = false; + goto unlock_and_out; + } + + /* device state needs to be ok */ + if (aq->dev_state != AP_DEV_STATE_OPERATING) { + rc = false; + goto unlock_and_out; + } + + /* SE guest's queues additionally need to be bound */ + if (ap_is_se_guest()) { + if (!ap_q_supported_in_se(aq)) { + rc = false; + goto unlock_and_out; + } + if (ap_q_needs_bind(aq) && + !(aq->se_bstate == AP_BS_Q_USABLE || + aq->se_bstate == AP_BS_Q_USABLE_NO_SECURE_KEY)) + rc = false; + } + +unlock_and_out: + spin_unlock_bh(&aq->lock); + return rc; +} +EXPORT_SYMBOL(ap_queue_usable); + +/** * ap_cancel_message(): Cancel a crypto request. * @aq: The AP device that has the message queued * @ap_msg: The message that is to be removed @@ -1143,7 +1359,7 @@ void ap_queue_prepare_remove(struct ap_queue *aq) /* move queue device state to SHUTDOWN in progress */ aq->dev_state = AP_DEV_STATE_SHUTDOWN; spin_unlock_bh(&aq->lock); - del_timer_sync(&aq->timeout); + timer_delete_sync(&aq->timeout); } void ap_queue_remove(struct ap_queue *aq) @@ -1160,14 +1376,19 @@ void ap_queue_remove(struct ap_queue *aq) spin_unlock_bh(&aq->lock); } -void ap_queue_init_state(struct ap_queue *aq) +void _ap_queue_init_state(struct ap_queue *aq) { - spin_lock_bh(&aq->lock); aq->dev_state = AP_DEV_STATE_OPERATING; aq->sm_state = AP_SM_STATE_RESET_START; aq->last_err_rc = 0; aq->assoc_idx = ASSOC_IDX_INVALID; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); +} + +void ap_queue_init_state(struct ap_queue *aq) +{ + spin_lock_bh(&aq->lock); + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); } EXPORT_SYMBOL(ap_queue_init_state); diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 6cfb6b2340c9..ad1cd699f53b 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -7,1333 +7,703 @@ * Author(s): Harald Freudenberger */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt -#include <linux/fs.h> #include <linux/init.h> #include <linux/miscdevice.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> -#include <linux/kallsyms.h> -#include <linux/debugfs.h> -#include <linux/random.h> -#include <linux/cpufeature.h> -#include <asm/zcrypt.h> -#include <asm/cpacf.h> -#include <asm/pkey.h> -#include <crypto/aes.h> #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" -#include "zcrypt_ep11misc.h" -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("IBM Corporation"); -MODULE_DESCRIPTION("s390 protected key interface"); - -#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ -#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header)) -#define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ -#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ -#define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ +#include "pkey_base.h" /* - * debug feature data and functions + * Helper functions */ - -static debug_info_t *debug_info; - -#define DEBUG_DBG(...) debug_sprintf_event(debug_info, 6, ##__VA_ARGS__) -#define DEBUG_INFO(...) debug_sprintf_event(debug_info, 5, ##__VA_ARGS__) -#define DEBUG_WARN(...) debug_sprintf_event(debug_info, 4, ##__VA_ARGS__) -#define DEBUG_ERR(...) debug_sprintf_event(debug_info, 3, ##__VA_ARGS__) - -static void __init pkey_debug_init(void) -{ - /* 5 arguments per dbf entry (including the format string ptr) */ - debug_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); - debug_register_view(debug_info, &debug_sprintf_view); - debug_set_level(debug_info, 3); -} - -static void __exit pkey_debug_exit(void) +static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags) { - debug_unregister(debug_info); -} + int rc; -/* inside view of a protected key token (only type 0x00 version 0x01) */ -struct protaeskeytoken { - u8 type; /* 0x00 for PAES specific key tokens */ - u8 res0[3]; - u8 version; /* should be 0x01 for protected AES key token */ - u8 res1[3]; - u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ - u32 len; /* bytes actually stored in protkey[] */ - u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ -} __packed; - -/* inside view of a clear key token (type 0x00 version 0x02) */ -struct clearkeytoken { - u8 type; /* 0x00 for PAES specific key tokens */ - u8 res0[3]; - u8 version; /* 0x02 for clear key token */ - u8 res1[3]; - u32 keytype; /* key type, one of the PKEY_KEYTYPE_* values */ - u32 len; /* bytes actually stored in clearkey[] */ - u8 clearkey[]; /* clear key value */ -} __packed; - -/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */ -static inline u32 pkey_keytype_aes_to_size(u32 keytype) -{ - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - return 16; - case PKEY_KEYTYPE_AES_192: - return 24; - case PKEY_KEYTYPE_AES_256: - return 32; - default: - return 0; + /* try the direct way */ + rc = pkey_handler_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype, xflags); + + /* if this did not work, try the slowpath way */ + if (rc == -ENODEV) { + rc = pkey_handler_slowpath_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype, xflags); + if (rc) + rc = -ENODEV; } + + pr_debug("rc=%d\n", rc); + return rc; } /* - * Create a protected key from a clear key value via PCKMO instruction. + * In-Kernel function: Transform a key blob (of any type) into a protected key */ -static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +int pkey_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags) { - /* mask of available pckmo subfunctions */ - static cpacf_mask_t pckmo_functions; - - u8 paramblock[112]; - u32 pkeytype; - int keysize; - long fc; - - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - /* 16 byte key, 32 byte aes wkvp, total 48 bytes */ - keysize = 16; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_128_KEY; - break; - case PKEY_KEYTYPE_AES_192: - /* 24 byte key, 32 byte aes wkvp, total 56 bytes */ - keysize = 24; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_192_KEY; - break; - case PKEY_KEYTYPE_AES_256: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_256_KEY; - break; - case PKEY_KEYTYPE_ECC_P256: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P256_KEY; - break; - case PKEY_KEYTYPE_ECC_P384: - /* 48 byte key, 32 byte aes wkvp, total 80 bytes */ - keysize = 48; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P384_KEY; - break; - case PKEY_KEYTYPE_ECC_P521: - /* 80 byte key, 32 byte aes wkvp, total 112 bytes */ - keysize = 80; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P521_KEY; - break; - case PKEY_KEYTYPE_ECC_ED25519: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY; - break; - case PKEY_KEYTYPE_ECC_ED448: - /* 64 byte key, 32 byte aes wkvp, total 96 bytes */ - keysize = 64; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; - break; - default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, keytype); - return -EINVAL; - } - - if (*protkeylen < keysize + AES_WK_VP_SIZE) { - DEBUG_ERR("%s prot key buffer size too small: %u < %d\n", - __func__, *protkeylen, keysize + AES_WK_VP_SIZE); - return -EINVAL; - } + int rc; - /* Did we already check for PCKMO ? */ - if (!pckmo_functions.bytes[0]) { - /* no, so check now */ - if (!cpacf_query(CPACF_PCKMO, &pckmo_functions)) - return -ENODEV; - } - /* check for the pckmo subfunction we need now */ - if (!cpacf_test_func(&pckmo_functions, fc)) { - DEBUG_ERR("%s pckmo functions not available\n", __func__); - return -ENODEV; + rc = key2protkey(NULL, 0, key, keylen, + protkey, protkeylen, protkeytype, xflags); + if (rc == -ENODEV) { + pkey_handler_request_modules(); + rc = key2protkey(NULL, 0, key, keylen, + protkey, protkeylen, protkeytype, xflags); } - /* prepare param block */ - memset(paramblock, 0, sizeof(paramblock)); - memcpy(paramblock, clrkey, keysize); - - /* call the pckmo instruction */ - cpacf_pckmo(fc, paramblock); - - /* copy created protected key to key buffer including the wkvp block */ - *protkeylen = keysize + AES_WK_VP_SIZE; - memcpy(protkey, paramblock, *protkeylen); - *protkeytype = pkeytype; - - return 0; + return rc; } +EXPORT_SYMBOL(pkey_key2protkey); /* - * Find card and transform secure key into protected key. + * Ioctl functions */ -static int pkey_skey2pkey(const u8 *key, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) + +static void *_copy_key_from_user(void __user *ukey, size_t keylen) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - u16 cardnr, domain; - int rc, verify; - - zcrypt_wait_api_operational(); - - /* - * The cca_xxx2protkey call may fail when a card has been - * addressed where the master key was changed after last fetch - * of the mkvp into the cache. Try 3 times: First without verify - * then with verify and last round with verify and old master - * key verification pattern match not ignored. - */ - for (verify = 0; verify < 3; verify++) { - rc = cca_findcard(key, &cardnr, &domain, verify); - if (rc < 0) - continue; - if (rc > 0 && verify < 2) - continue; - switch (hdr->version) { - case TOKVER_CCA_AES: - rc = cca_sec2protkey(cardnr, domain, key, - protkey, protkeylen, protkeytype); - break; - case TOKVER_CCA_VLSC: - rc = cca_cipher2protkey(cardnr, domain, key, - protkey, protkeylen, - protkeytype); - break; - default: - return -EINVAL; - } - if (rc == 0) - break; - } + if (!ukey || keylen < MINKEYBLOBBUFSIZE || keylen > KEYBLOBBUFSIZE) + return ERR_PTR(-EINVAL); - if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + return memdup_user(ukey, keylen); +} - return rc; +static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) +{ + if (!uapqns || nr_apqns == 0) + return NULL; + + return memdup_array_user(uapqns, nr_apqns, sizeof(struct pkey_apqn)); } -/* - * Construct EP11 key with given clear key value. - */ -static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, - u8 *keybuf, size_t *keybuflen) +static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) { - u32 nr_apqns, *apqns = NULL; - u16 card, dom; - int i, rc; - - zcrypt_wait_api_operational(); - - /* build a list of apqns suitable for ep11 keys with cpacf support */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - NULL); - if (rc) - goto out; - - /* go through the list of apqns and try to bild an ep11 key */ - for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_clr2keyblob(card, dom, clrkeylen * 8, - 0, clrkey, keybuf, keybuflen, - PKEY_TYPE_EP11); - if (rc == 0) - break; - } + struct pkey_genseck kgs; + struct pkey_apqn apqn; + u32 keybuflen; + int rc; + + if (copy_from_user(&kgs, ugs, sizeof(kgs))) + return -EFAULT; + + apqn.card = kgs.cardnr; + apqn.domain = kgs.domain; + keybuflen = sizeof(kgs.seckey.seckey); + rc = pkey_handler_gen_key(&apqn, 1, + kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kgs.seckey.seckey, &keybuflen, NULL, 0); + pr_debug("gen_key()=%d\n", rc); + if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) + rc = -EFAULT; + memzero_explicit(&kgs, sizeof(kgs)); -out: - kfree(apqns); - if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); return rc; } -/* - * Find card and transform EP11 secure key into protected key. - */ -static int pkey_ep11key2pkey(const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) { - u32 nr_apqns, *apqns = NULL; - u16 card, dom; - int i, rc; - - zcrypt_wait_api_operational(); - - /* build a list of apqns suitable for this key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - /* go through the list of apqns and try to derive an pkey */ - for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, protkeytype); - if (rc == 0) - break; - } + struct pkey_clr2seck kcs; + struct pkey_apqn apqn; + u32 keybuflen; + int rc; + + if (copy_from_user(&kcs, ucs, sizeof(kcs))) + return -EFAULT; + + apqn.card = kcs.cardnr; + apqn.domain = kcs.domain; + keybuflen = sizeof(kcs.seckey.seckey); + rc = pkey_handler_clr_to_key(&apqn, 1, + kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kcs.clrkey.clrkey, + pkey_keytype_aes_to_size(kcs.keytype), + kcs.seckey.seckey, &keybuflen, NULL, 0); + pr_debug("clr_to_key()=%d\n", rc); + if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) + rc = -EFAULT; + memzero_explicit(&kcs, sizeof(kcs)); -out: - kfree(apqns); - if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); return rc; } -/* - * Verify key and give back some info about the key. - */ -static int pkey_verifykey(const struct pkey_seckey *seckey, - u16 *pcardnr, u16 *pdomain, - u16 *pkeysize, u32 *pattributes) +static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp) { - struct secaeskeytoken *t = (struct secaeskeytoken *)seckey; - u16 cardnr, domain; + struct pkey_sec2protk ksp; + struct pkey_apqn apqn; int rc; - /* check the secure key for valid AES secure key */ - rc = cca_check_secaeskeytoken(debug_info, 3, (u8 *)seckey, 0); - if (rc) - goto out; - if (pattributes) - *pattributes = PKEY_VERIFY_ATTR_AES; - if (pkeysize) - *pkeysize = t->bitsize; - - /* try to find a card which can handle this key */ - rc = cca_findcard(seckey->seckey, &cardnr, &domain, 1); - if (rc < 0) - goto out; - - if (rc > 0) { - /* key mkvp matches to old master key mkvp */ - DEBUG_DBG("%s secure key has old mkvp\n", __func__); - if (pattributes) - *pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP; - rc = 0; - } - - if (pcardnr) - *pcardnr = cardnr; - if (pdomain) - *pdomain = domain; + if (copy_from_user(&ksp, usp, sizeof(ksp))) + return -EFAULT; + + apqn.card = ksp.cardnr; + apqn.domain = ksp.domain; + ksp.protkey.len = sizeof(ksp.protkey.protkey); + rc = pkey_handler_key_to_protkey(&apqn, 1, + ksp.seckey.seckey, + sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type, + 0); + pr_debug("key_to_protkey()=%d\n", rc); + if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) + rc = -EFAULT; + memzero_explicit(&ksp, sizeof(ksp)); -out: - DEBUG_DBG("%s rc=%d\n", __func__, rc); return rc; } -/* - * Generate a random protected key - */ -static int pkey_genprotkey(u32 keytype, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) { - u8 clrkey[32]; - int keysize; + struct pkey_clr2protk kcp; + struct clearkeytoken *t; + u32 keylen; + u8 *tmpbuf; int rc; - keysize = pkey_keytype_aes_to_size(keytype); - if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); + if (copy_from_user(&kcp, ucp, sizeof(kcp))) + return -EFAULT; + + /* build a 'clear key token' from the clear key value */ + keylen = pkey_keytype_aes_to_size(kcp.keytype); + if (!keylen) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, kcp.keytype); + memzero_explicit(&kcp, sizeof(kcp)); return -EINVAL; } + tmpbuf = kzalloc(sizeof(*t) + keylen, GFP_KERNEL); + if (!tmpbuf) { + memzero_explicit(&kcp, sizeof(kcp)); + return -ENOMEM; + } + t = (struct clearkeytoken *)tmpbuf; + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_CLEAR_KEY; + t->keytype = (keylen - 8) >> 3; + t->len = keylen; + memcpy(t->clearkey, kcp.clrkey.clrkey, keylen); + kcp.protkey.len = sizeof(kcp.protkey.protkey); - /* generate a dummy random clear key */ - get_random_bytes(clrkey, keysize); + rc = key2protkey(NULL, 0, + tmpbuf, sizeof(*t) + keylen, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type, 0); + pr_debug("key2protkey()=%d\n", rc); - /* convert it to a dummy protected key */ - rc = pkey_clr2protkey(keytype, clrkey, - protkey, protkeylen, protkeytype); - if (rc) - return rc; + kfree_sensitive(tmpbuf); - /* replace the key part of the protected key with random bytes */ - get_random_bytes(protkey, keysize); + if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) + rc = -EFAULT; + memzero_explicit(&kcp, sizeof(kcp)); - return 0; + return rc; } -/* - * Verify if a protected key is still valid - */ -static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, - u32 protkeytype) +static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) { - struct { - u8 iv[AES_BLOCK_SIZE]; - u8 key[MAXPROTKEYSIZE]; - } param; - u8 null_msg[AES_BLOCK_SIZE]; - u8 dest_buf[AES_BLOCK_SIZE]; - unsigned int k, pkeylen; - unsigned long fc; - - switch (protkeytype) { - case PKEY_KEYTYPE_AES_128: - pkeylen = 16 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_128; - break; - case PKEY_KEYTYPE_AES_192: - pkeylen = 24 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_192; - break; - case PKEY_KEYTYPE_AES_256: - pkeylen = 32 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_256; - break; - default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, - protkeytype); - return -EINVAL; - } - if (protkeylen != pkeylen) { - DEBUG_ERR("%s invalid protected key size %u for keytype %u\n", - __func__, protkeylen, protkeytype); - return -EINVAL; - } + struct pkey_findcard kfc; + struct pkey_apqn *apqns; + size_t nr_apqns; + int rc; - memset(null_msg, 0, sizeof(null_msg)); + if (copy_from_user(&kfc, ufc, sizeof(kfc))) + return -EFAULT; - memset(param.iv, 0, sizeof(param.iv)); - memcpy(param.key, protkey, protkeylen); + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) + return -ENOMEM; - k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, - sizeof(null_msg)); - if (k != sizeof(null_msg)) { - DEBUG_ERR("%s protected key is not valid\n", __func__); - return -EKEYREJECTED; + rc = pkey_handler_apqns_for_key(kfc.seckey.seckey, + sizeof(kfc.seckey.seckey), + PKEY_FLAGS_MATCH_CUR_MKVP, + apqns, &nr_apqns, 0); + if (rc == -ENODEV) + rc = pkey_handler_apqns_for_key(kfc.seckey.seckey, + sizeof(kfc.seckey.seckey), + PKEY_FLAGS_MATCH_ALT_MKVP, + apqns, &nr_apqns, 0); + pr_debug("apqns_for_key()=%d\n", rc); + if (rc) { + kfree(apqns); + return rc; } + kfc.cardnr = apqns[0].card; + kfc.domain = apqns[0].domain; + kfree(apqns); + if (copy_to_user(ufc, &kfc, sizeof(kfc))) + return -EFAULT; return 0; } -/* Helper for pkey_nonccatok2pkey, handles aes clear key token */ -static int nonccatokaes2pkey(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp) { - size_t tmpbuflen = max_t(size_t, SECKEYBLOBSIZE, MAXEP11AESKEYBLOBSIZE); - u8 *tmpbuf = NULL; - u32 keysize; + struct pkey_skey2pkey ksp; int rc; - keysize = pkey_keytype_aes_to_size(t->keytype); - if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); - return -EINVAL; - } - if (t->len != keysize) { - DEBUG_ERR("%s non clear key aes token: invalid key len %u\n", - __func__, t->len); - return -EINVAL; - } + if (copy_from_user(&ksp, usp, sizeof(ksp))) + return -EFAULT; + + ksp.protkey.len = sizeof(ksp.protkey.protkey); + rc = pkey_handler_key_to_protkey(NULL, 0, + ksp.seckey.seckey, + sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, + &ksp.protkey.type, 0); + pr_debug("key_to_protkey()=%d\n", rc); + if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) + rc = -EFAULT; + memzero_explicit(&ksp, sizeof(ksp)); - /* try direct way with the PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, t->clearkey, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; - - /* PCKMO failed, so try the CCA secure key way */ - tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); - if (!tmpbuf) - return -ENOMEM; - zcrypt_wait_api_operational(); - rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, t->clearkey, tmpbuf); - if (rc) - goto try_via_ep11; - rc = pkey_skey2pkey(tmpbuf, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; - -try_via_ep11: - /* if the CCA way also failed, let's try via EP11 */ - rc = pkey_clr2ep11key(t->clearkey, t->len, - tmpbuf, &tmpbuflen); - if (rc) - goto failure; - rc = pkey_ep11key2pkey(tmpbuf, tmpbuflen, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; - -failure: - DEBUG_ERR("%s unable to build protected key from clear", __func__); - -out: - kfree(tmpbuf); return rc; } -/* Helper for pkey_nonccatok2pkey, handles ecc clear key token */ -static int nonccatokecc2pkey(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_verifykey(struct pkey_verifykey __user *uvk) { - u32 keylen; + u32 keytype, keybitsize, flags; + struct pkey_verifykey kvk; int rc; - switch (t->keytype) { - case PKEY_KEYTYPE_ECC_P256: - keylen = 32; - break; - case PKEY_KEYTYPE_ECC_P384: - keylen = 48; - break; - case PKEY_KEYTYPE_ECC_P521: - keylen = 80; - break; - case PKEY_KEYTYPE_ECC_ED25519: - keylen = 32; - break; - case PKEY_KEYTYPE_ECC_ED448: - keylen = 64; - break; - default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); - return -EINVAL; - } + if (copy_from_user(&kvk, uvk, sizeof(kvk))) + return -EFAULT; - if (t->len != keylen) { - DEBUG_ERR("%s non clear key ecc token: invalid key len %u\n", - __func__, t->len); - return -EINVAL; - } - - /* only one path possible: via PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, t->clearkey, - protkey, protkeylen, protkeytype); - if (rc) { - DEBUG_ERR("%s unable to build protected key from clear", - __func__); - } + kvk.cardnr = 0xFFFF; + kvk.domain = 0xFFFF; + rc = pkey_handler_verify_key(kvk.seckey.seckey, + sizeof(kvk.seckey.seckey), + &kvk.cardnr, &kvk.domain, + &keytype, &keybitsize, &flags, 0); + pr_debug("verify_key()=%d\n", rc); + if (!rc && keytype != PKEY_TYPE_CCA_DATA) + rc = -EINVAL; + kvk.attributes = PKEY_VERIFY_ATTR_AES; + kvk.keysize = (u16)keybitsize; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + kvk.attributes |= PKEY_VERIFY_ATTR_OLD_MKVP; + if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) + rc = -EFAULT; + memzero_explicit(&kvk, sizeof(kvk)); return rc; } -/* - * Transform a non-CCA key token into a protected key - */ -static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc = -EINVAL; + struct pkey_genprotk kgp; + int rc; - switch (hdr->version) { - case TOKVER_PROTECTED_KEY: { - struct protaeskeytoken *t; + if (copy_from_user(&kgp, ugp, sizeof(kgp))) + return -EFAULT; - if (keylen != sizeof(struct protaeskeytoken)) - goto out; - t = (struct protaeskeytoken *)key; - rc = pkey_verifyprotkey(t->protkey, t->len, t->keytype); - if (rc) - goto out; - memcpy(protkey, t->protkey, t->len); - *protkeylen = t->len; - *protkeytype = t->keytype; - break; - } - case TOKVER_CLEAR_KEY: { - struct clearkeytoken *t = (struct clearkeytoken *)key; - - if (keylen < sizeof(struct clearkeytoken) || - keylen != sizeof(*t) + t->len) - goto out; - switch (t->keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - rc = nonccatokaes2pkey(t, protkey, - protkeylen, protkeytype); - break; - case PKEY_KEYTYPE_ECC_P256: - case PKEY_KEYTYPE_ECC_P384: - case PKEY_KEYTYPE_ECC_P521: - case PKEY_KEYTYPE_ECC_ED25519: - case PKEY_KEYTYPE_ECC_ED448: - rc = nonccatokecc2pkey(t, protkey, - protkeylen, protkeytype); - break; - default: - DEBUG_ERR("%s unknown/unsupported non cca clear key type %u\n", - __func__, t->keytype); - return -EINVAL; - } - break; - } - case TOKVER_EP11_AES: { - /* check ep11 key for exportable as protected key */ - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); - if (rc) - goto out; - rc = pkey_ep11key2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - } - case TOKVER_EP11_AES_WITH_HEADER: - /* check ep11 key with header for exportable as protected key */ - rc = ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1); - if (rc) - goto out; - rc = pkey_ep11key2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - default: - DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", - __func__, hdr->version); - } + kgp.protkey.len = sizeof(kgp.protkey.protkey); + rc = pkey_handler_gen_key(NULL, 0, kgp.keytype, + PKEY_TYPE_PROTKEY, 0, 0, + kgp.protkey.protkey, &kgp.protkey.len, + &kgp.protkey.type, 0); + pr_debug("gen_key()=%d\n", rc); + if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) + rc = -EFAULT; + memzero_explicit(&kgp, sizeof(kgp)); -out: return rc; } -/* - * Transform a CCA internal key token into a protected key - */ -static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) { - struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_verifyprotk kvp; + struct protaeskeytoken *t; + u32 keytype; + u8 *tmpbuf; + int rc; - switch (hdr->version) { - case TOKVER_CCA_AES: - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - break; - case TOKVER_CCA_VLSC: - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - break; - default: - DEBUG_ERR("%s unknown/unsupported CCA internal token version %d\n", - __func__, hdr->version); + if (copy_from_user(&kvp, uvp, sizeof(kvp))) + return -EFAULT; + + keytype = pkey_aes_bitsize_to_keytype(8 * kvp.protkey.len); + if (!keytype) { + PKEY_DBF_ERR("%s unknown/unsupported protkey length %u\n", + __func__, kvp.protkey.len); + memzero_explicit(&kvp, sizeof(kvp)); return -EINVAL; } - return pkey_skey2pkey(key, protkey, protkeylen, protkeytype); + /* build a 'protected key token' from the raw protected key */ + tmpbuf = kzalloc(sizeof(*t), GFP_KERNEL); + if (!tmpbuf) { + memzero_explicit(&kvp, sizeof(kvp)); + return -ENOMEM; + } + t = (struct protaeskeytoken *)tmpbuf; + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + t->len = kvp.protkey.len; + memcpy(t->protkey, kvp.protkey.protkey, kvp.protkey.len); + + rc = pkey_handler_verify_key(tmpbuf, sizeof(*t), + NULL, NULL, NULL, NULL, NULL, 0); + pr_debug("verify_key()=%d\n", rc); + + kfree_sensitive(tmpbuf); + memzero_explicit(&kvp, sizeof(kvp)); + + return rc; } -/* - * Transform a key blob (of any type) into a protected key - */ -int pkey_keyblob2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_kblob2protk(struct pkey_kblob2pkey __user *utp) { - struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_kblob2pkey ktp; + u8 *kkey; int rc; - if (keylen < sizeof(struct keytoken_header)) { - DEBUG_ERR("%s invalid keylen %d\n", __func__, keylen); - return -EINVAL; - } - - switch (hdr->type) { - case TOKTYPE_NON_CCA: - rc = pkey_nonccatok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - case TOKTYPE_CCA_INTERNAL: - rc = pkey_ccainttok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - default: - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; - } + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) + return PTR_ERR(kkey); + ktp.protkey.len = sizeof(ktp.protkey.protkey); + rc = key2protkey(NULL, 0, kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type, 0); + pr_debug("key2protkey()=%d\n", rc); + kfree_sensitive(kkey); + if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) + rc = -EFAULT; + memzero_explicit(&ktp, sizeof(ktp)); - DEBUG_DBG("%s rc=%d\n", __func__, rc); return rc; } -EXPORT_SYMBOL(pkey_keyblob2pkey); -static int pkey_genseckey2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type ktype, enum pkey_key_size ksize, - u32 kflags, u8 *keybuf, size_t *keybufsize) +static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) { - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; + u32 klen = KEYBLOBBUFSIZE; + struct pkey_genseck2 kgs; + struct pkey_apqn *apqns; + u8 *kkey; + int rc; + u32 u; - /* check key type and size */ - switch (ktype) { - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - if (*keybufsize < SECKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11: - if (*keybufsize < MINEP11AESKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11_AES: - if (*keybufsize < (sizeof(struct ep11kblob_header) + - MINEP11AESKEYBLOBSIZE)) - return -EINVAL; - break; - default: + if (copy_from_user(&kgs, ugs, sizeof(kgs))) + return -EFAULT; + u = pkey_aes_bitsize_to_keytype(kgs.size); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kgs.size); return -EINVAL; } - switch (ksize) { - case PKEY_SIZE_AES_128: - case PKEY_SIZE_AES_192: - case PKEY_SIZE_AES_256: - break; - default: - return -EINVAL; + apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = kzalloc(klen, GFP_KERNEL); + if (!kkey) { + kfree(apqns); + return -ENOMEM; } - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES) { - rc = ep11_genaeskey(card, dom, ksize, kflags, - keybuf, keybufsize, ktype); - } else if (ktype == PKEY_TYPE_CCA_DATA) { - rc = cca_genseckey(card, dom, ksize, keybuf); - *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); - } else { - /* TOKVER_CCA_VLSC */ - rc = cca_gencipherkey(card, dom, ksize, kflags, - keybuf, keybufsize); + rc = pkey_handler_gen_key(apqns, kgs.apqn_entries, + u, kgs.type, kgs.size, kgs.keygenflags, + kkey, &klen, NULL, 0); + pr_debug("gen_key()=%d\n", rc); + kfree(apqns); + if (rc) { + kfree_sensitive(kkey); + return rc; + } + if (kgs.key) { + if (kgs.keylen < klen) { + kfree_sensitive(kkey); + return -EINVAL; + } + if (copy_to_user(kgs.key, kkey, klen)) { + kfree_sensitive(kkey); + return -EFAULT; } - if (rc == 0) - break; } + kgs.keylen = klen; + if (copy_to_user(ugs, &kgs, sizeof(kgs))) + rc = -EFAULT; + kfree_sensitive(kkey); return rc; } -static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type ktype, enum pkey_key_size ksize, - u32 kflags, const u8 *clrkey, - u8 *keybuf, size_t *keybufsize) +static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) { - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - /* check key type and size */ - switch (ktype) { - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - if (*keybufsize < SECKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11: - if (*keybufsize < MINEP11AESKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11_AES: - if (*keybufsize < (sizeof(struct ep11kblob_header) + - MINEP11AESKEYBLOBSIZE)) - return -EINVAL; - break; - default: + u32 klen = KEYBLOBBUFSIZE; + struct pkey_clr2seck2 kcs; + struct pkey_apqn *apqns; + u8 *kkey; + int rc; + u32 u; + + if (copy_from_user(&kcs, ucs, sizeof(kcs))) + return -EFAULT; + u = pkey_aes_bitsize_to_keytype(kcs.size); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kcs.size); + memzero_explicit(&kcs, sizeof(kcs)); return -EINVAL; } - switch (ksize) { - case PKEY_SIZE_AES_128: - case PKEY_SIZE_AES_192: - case PKEY_SIZE_AES_256: - break; - default: - return -EINVAL; + apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries); + if (IS_ERR(apqns)) { + memzero_explicit(&kcs, sizeof(kcs)); + return PTR_ERR(apqns); } - - zcrypt_wait_api_operational(); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES) { - rc = ep11_clr2keyblob(card, dom, ksize, kflags, - clrkey, keybuf, keybufsize, - ktype); - } else if (ktype == PKEY_TYPE_CCA_DATA) { - rc = cca_clr2seckey(card, dom, ksize, - clrkey, keybuf); - *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); - } else { - /* TOKVER_CCA_VLSC */ - rc = cca_clr2cipherkey(card, dom, ksize, kflags, - clrkey, keybuf, keybufsize); + kkey = kzalloc(klen, GFP_KERNEL); + if (!kkey) { + kfree(apqns); + memzero_explicit(&kcs, sizeof(kcs)); + return -ENOMEM; + } + rc = pkey_handler_clr_to_key(apqns, kcs.apqn_entries, + u, kcs.type, kcs.size, kcs.keygenflags, + kcs.clrkey.clrkey, kcs.size / 8, + kkey, &klen, NULL, 0); + pr_debug("clr_to_key()=%d\n", rc); + kfree(apqns); + if (rc) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return rc; + } + if (kcs.key) { + if (kcs.keylen < klen) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return -EINVAL; + } + if (copy_to_user(kcs.key, kkey, klen)) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return -EFAULT; } - if (rc == 0) - break; } + kcs.keylen = klen; + if (copy_to_user(ucs, &kcs, sizeof(kcs))) + rc = -EFAULT; + memzero_explicit(&kcs, sizeof(kcs)); + kfree_sensitive(kkey); return rc; } -static int pkey_verifykey2(const u8 *key, size_t keylen, - u16 *cardnr, u16 *domain, - enum pkey_key_type *ktype, - enum pkey_key_size *ksize, u32 *flags) +static int pkey_ioctl_verifykey2(struct pkey_verifykey2 __user *uvk) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 _nr_apqns, *_apqns = NULL; + struct pkey_verifykey2 kvk; + u8 *kkey; int rc; - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; - - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - struct secaeskeytoken *t = (struct secaeskeytoken *)key; + if (copy_from_user(&kvk, uvk, sizeof(kvk))) + return -EFAULT; + kkey = _copy_key_from_user(kvk.key, kvk.keylen); + if (IS_ERR(kkey)) + return PTR_ERR(kkey); - rc = cca_check_secaeskeytoken(debug_info, 3, key, 0); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_CCA_DATA; - if (ksize) - *ksize = (enum pkey_key_size)t->bitsize; - - rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX3C, AES_MK_SET, t->mkvp, 0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&_apqns, &_nr_apqns, - *cardnr, *domain, - ZCRYPT_CEX3C, AES_MK_SET, - 0, t->mkvp, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; - } - if (rc) - goto out; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - struct cipherkeytoken *t = (struct cipherkeytoken *)key; - - rc = cca_check_secaescipherkey(debug_info, 3, key, 0, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_CCA_CIPHER; - if (ksize) { - *ksize = PKEY_SIZE_UNKNOWN; - if (!t->plfver && t->wpllen == 512) - *ksize = PKEY_SIZE_AES_128; - else if (!t->plfver && t->wpllen == 576) - *ksize = PKEY_SIZE_AES_192; - else if (!t->plfver && t->wpllen == 640) - *ksize = PKEY_SIZE_AES_256; - } - - rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX6, AES_MK_SET, t->mkvp0, 0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&_apqns, &_nr_apqns, - *cardnr, *domain, - ZCRYPT_CEX6, AES_MK_SET, - 0, t->mkvp0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; - } - if (rc) - goto out; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES) { - struct ep11keyblob *kb = (struct ep11keyblob *)key; - int api; - - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_EP11; - if (ksize) - *ksize = kb->head.bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - if (flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; + rc = pkey_handler_verify_key(kkey, kvk.keylen, + &kvk.cardnr, &kvk.domain, + &kvk.type, &kvk.size, &kvk.flags, 0); + pr_debug("verify_key()=%d\n", rc); - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - struct ep11kblob_header *kh = (struct ep11kblob_header *)key; - int api; + kfree_sensitive(kkey); + if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) + return -EFAULT; - rc = ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_EP11_AES; - if (ksize) - *ksize = kh->bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; + return rc; +} - if (flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; +static int pkey_ioctl_kblob2protk2(struct pkey_kblob2pkey2 __user *utp) +{ + struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey2 ktp; + u8 *kkey; + int rc; - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - } else { - rc = -EINVAL; + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); } + ktp.protkey.len = sizeof(ktp.protkey.protkey); + rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type, 0); + pr_debug("key2protkey()=%d\n", rc); + kfree(apqns); + kfree_sensitive(kkey); + if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) + rc = -EFAULT; + memzero_explicit(&ktp, sizeof(ktp)); -out: - kfree(_apqns); return rc; } -static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pkey_ioctl_apqns4k(struct pkey_apqns4key __user *uak) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; + struct pkey_apqn *apqns = NULL; + struct pkey_apqns4key kak; + size_t nr_apqns, len; + u8 *kkey; + int rc; - if (hdr->type == TOKTYPE_CCA_INTERNAL) { - if (hdr->version == TOKVER_CCA_AES) { - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) - return -EINVAL; - } else if (hdr->version == TOKVER_CCA_VLSC) { - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) - return -EINVAL; - } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); + if (copy_from_user(&kak, uak, sizeof(kak))) + return -EFAULT; + nr_apqns = kak.apqn_entries; + if (nr_apqns) { + apqns = kmalloc_array(nr_apqns, + sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!apqns) + return -ENOMEM; + } + kkey = _copy_key_from_user(kak.key, kak.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); + } + rc = pkey_handler_apqns_for_key(kkey, kak.keylen, kak.flags, + apqns, &nr_apqns, 0); + pr_debug("apqns_for_key()=%d\n", rc); + kfree_sensitive(kkey); + if (rc && rc != -ENOSPC) { + kfree(apqns); + return rc; + } + if (!rc && kak.apqns) { + if (nr_apqns > kak.apqn_entries) { + kfree(apqns); return -EINVAL; } - } else if (hdr->type == TOKTYPE_NON_CCA) { - if (hdr->version == TOKVER_EP11_AES) { - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - if (ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1)) - return -EINVAL; - } else { - return pkey_nonccatok2pkey(key, keylen, - protkey, protkeylen, - protkeytype); - } - } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - rc = cca_sec2protkey(card, dom, key, - protkey, protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - rc = cca_cipher2protkey(card, dom, key, - protkey, protkeylen, - protkeytype); - } else { - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, - protkeytype); + len = nr_apqns * sizeof(struct pkey_apqn); + if (len) { + if (copy_to_user(kak.apqns, apqns, len)) { + kfree(apqns); + return -EFAULT; + } } - if (rc == 0) - break; } + kak.apqn_entries = nr_apqns; + if (copy_to_user(uak, &kak, sizeof(kak))) + rc = -EFAULT; + kfree(apqns); return rc; } -static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int pkey_ioctl_apqns4kt(struct pkey_apqns4keytype __user *uat) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 _nr_apqns, *_apqns = NULL; + struct pkey_apqn *apqns = NULL; + struct pkey_apqns4keytype kat; + size_t nr_apqns, len; int rc; - if (keylen < sizeof(struct keytoken_header) || flags == 0) - return -EINVAL; - - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_NON_CCA && - (hdr->version == TOKVER_EP11_AES_WITH_HEADER || - hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - struct ep11keyblob *kb = (struct ep11keyblob *) - (key + sizeof(struct ep11kblob_header)); - int minhwtype = 0, api = 0; - - if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) - return -EINVAL; - if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { - minhwtype = ZCRYPT_CEX7; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - } - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, api, kb->wkvp); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) { - struct ep11keyblob *kb = (struct ep11keyblob *)key; - int minhwtype = 0, api = 0; - - if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) - return -EINVAL; - if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { - minhwtype = ZCRYPT_CEX7; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - } - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, api, kb->wkvp); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { - u64 cur_mkvp = 0, old_mkvp = 0; - int minhwtype = ZCRYPT_CEX3C; - - if (hdr->version == TOKVER_CCA_AES) { - struct secaeskeytoken *t = (struct secaeskeytoken *)key; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp; - } else if (hdr->version == TOKVER_CCA_VLSC) { - struct cipherkeytoken *t = (struct cipherkeytoken *)key; - - minhwtype = ZCRYPT_CEX6; - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp0; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp0; - } else { - /* unknown cca internal token type */ + if (copy_from_user(&kat, uat, sizeof(kat))) + return -EFAULT; + nr_apqns = kat.apqn_entries; + if (nr_apqns) { + apqns = kmalloc_array(nr_apqns, + sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!apqns) + return -ENOMEM; + } + rc = pkey_handler_apqns_for_keytype(kat.type, + kat.cur_mkvp, kat.alt_mkvp, + kat.flags, apqns, &nr_apqns, 0); + pr_debug("apqns_for_keytype()=%d\n", rc); + if (rc && rc != -ENOSPC) { + kfree(apqns); + return rc; + } + if (!rc && kat.apqns) { + if (nr_apqns > kat.apqn_entries) { + kfree(apqns); return -EINVAL; } - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, AES_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; - u64 cur_mkvp = 0, old_mkvp = 0; - - if (t->secid == 0x20) { - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp; - } else { - /* unknown cca internal 2 token type */ - return -EINVAL; + len = nr_apqns * sizeof(struct pkey_apqn); + if (len) { + if (copy_to_user(kat.apqns, apqns, len)) { + kfree(apqns); + return -EFAULT; + } } - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, APKA_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else { - return -EINVAL; } + kat.apqn_entries = nr_apqns; + if (copy_to_user(uat, &kat, sizeof(kat))) + rc = -EFAULT; + kfree(apqns); - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); - } - *nr_apqns = _nr_apqns; - -out: - kfree(_apqns); return rc; } -static int pkey_apqns4keytype(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int pkey_ioctl_kblob2protk3(struct pkey_kblob2pkey3 __user *utp) { - u32 _nr_apqns, *_apqns = NULL; + u32 protkeylen = PROTKEYBLOBBUFSIZE; + struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey3 ktp; + u8 *kkey, *protkey; int rc; - zcrypt_wait_api_operational(); - - if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { - u64 cur_mkvp = 0, old_mkvp = 0; - int minhwtype = ZCRYPT_CEX3C; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = *((u64 *)cur_mkvp); - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = *((u64 *)alt_mkvp); - if (ktype == PKEY_TYPE_CCA_CIPHER) - minhwtype = ZCRYPT_CEX6; - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, AES_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else if (ktype == PKEY_TYPE_CCA_ECC) { - u64 cur_mkvp = 0, old_mkvp = 0; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = *((u64 *)cur_mkvp); - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = *((u64 *)alt_mkvp); - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, APKA_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - - } else if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES || - ktype == PKEY_TYPE_EP11_ECC) { - u8 *wkvp = NULL; - int api; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - wkvp = cur_mkvp; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, api, wkvp); - if (rc) - goto out; - - } else { - return -EINVAL; + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); } - - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + protkey = kmalloc(protkeylen, GFP_KERNEL); + if (!protkey) { + kfree(apqns); + kfree_sensitive(kkey); + return -ENOMEM; } - *nr_apqns = _nr_apqns; - -out: - kfree(_apqns); - return rc; -} - -static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; - - if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 AES key blob with header */ - if (ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_ECC_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 ECC key blob with header */ - if (ep11_check_ecc_key_with_hdr(debug_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) { - /* EP11 AES key blob with header in session field */ - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { - if (hdr->version == TOKVER_CCA_AES) { - /* CCA AES data key */ - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) - return -EINVAL; - } else if (hdr->version == TOKVER_CCA_VLSC) { - /* CCA AES cipher key */ - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) - return -EINVAL; - } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); - return -EINVAL; - } - } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - /* CCA ECC (private) key */ - if (keylen < sizeof(struct eccprivkeytoken)) - return -EINVAL; - if (cca_check_sececckeytoken(debug_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA) { - return pkey_nonccatok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; + rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen, + protkey, &protkeylen, &ktp.pkeytype, 0); + pr_debug("key2protkey()=%d\n", rc); + kfree(apqns); + kfree_sensitive(kkey); + if (rc) { + kfree_sensitive(protkey); + return rc; } - - /* simple try all apqns from the list */ - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (hdr->type == TOKTYPE_NON_CCA && - (hdr->version == TOKVER_EP11_AES_WITH_HEADER || - hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); - else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) - rc = cca_sec2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) - rc = cca_cipher2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) - rc = cca_ecc2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else + if (ktp.pkey && ktp.pkeylen) { + if (protkeylen > ktp.pkeylen) { + kfree_sensitive(protkey); return -EINVAL; + } + if (copy_to_user(ktp.pkey, protkey, protkeylen)) { + kfree_sensitive(protkey); + return -EFAULT; + } } + kfree_sensitive(protkey); + ktp.pkeylen = protkeylen; + if (copy_to_user(utp, &ktp, sizeof(ktp))) + return -EFAULT; - return rc; -} - -/* - * File io functions - */ - -static void *_copy_key_from_user(void __user *ukey, size_t keylen) -{ - if (!ukey || keylen < MINKEYBLOBBUFSIZE || keylen > KEYBLOBBUFSIZE) - return ERR_PTR(-EINVAL); - - return memdup_user(ukey, keylen); -} - -static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) -{ - if (!uapqns || nr_apqns == 0) - return NULL; - - return memdup_user(uapqns, nr_apqns * sizeof(struct pkey_apqn)); + return 0; } static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, @@ -1342,445 +712,57 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, int rc; switch (cmd) { - case PKEY_GENSECK: { - struct pkey_genseck __user *ugs = (void __user *)arg; - struct pkey_genseck kgs; - - if (copy_from_user(&kgs, ugs, sizeof(kgs))) - return -EFAULT; - rc = cca_genseckey(kgs.cardnr, kgs.domain, - kgs.keytype, kgs.seckey.seckey); - DEBUG_DBG("%s cca_genseckey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(ugs, &kgs, sizeof(kgs))) - return -EFAULT; + case PKEY_GENSECK: + rc = pkey_ioctl_genseck((struct pkey_genseck __user *)arg); break; - } - case PKEY_CLR2SECK: { - struct pkey_clr2seck __user *ucs = (void __user *)arg; - struct pkey_clr2seck kcs; - - if (copy_from_user(&kcs, ucs, sizeof(kcs))) - return -EFAULT; - rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, - kcs.clrkey.clrkey, kcs.seckey.seckey); - DEBUG_DBG("%s cca_clr2seckey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(ucs, &kcs, sizeof(kcs))) - return -EFAULT; - memzero_explicit(&kcs, sizeof(kcs)); + case PKEY_CLR2SECK: + rc = pkey_ioctl_clr2seck((struct pkey_clr2seck __user *)arg); break; - } - case PKEY_SEC2PROTK: { - struct pkey_sec2protk __user *usp = (void __user *)arg; - struct pkey_sec2protk ksp; - - if (copy_from_user(&ksp, usp, sizeof(ksp))) - return -EFAULT; - ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = cca_sec2protkey(ksp.cardnr, ksp.domain, - ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s cca_sec2protkey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(usp, &ksp, sizeof(ksp))) - return -EFAULT; + case PKEY_SEC2PROTK: + rc = pkey_ioctl_sec2protk((struct pkey_sec2protk __user *)arg); break; - } - case PKEY_CLR2PROTK: { - struct pkey_clr2protk __user *ucp = (void __user *)arg; - struct pkey_clr2protk kcp; - - if (copy_from_user(&kcp, ucp, sizeof(kcp))) - return -EFAULT; - kcp.protkey.len = sizeof(kcp.protkey.protkey); - rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, - kcp.protkey.protkey, - &kcp.protkey.len, &kcp.protkey.type); - DEBUG_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(ucp, &kcp, sizeof(kcp))) - return -EFAULT; - memzero_explicit(&kcp, sizeof(kcp)); + case PKEY_CLR2PROTK: + rc = pkey_ioctl_clr2protk((struct pkey_clr2protk __user *)arg); break; - } - case PKEY_FINDCARD: { - struct pkey_findcard __user *ufc = (void __user *)arg; - struct pkey_findcard kfc; - - if (copy_from_user(&kfc, ufc, sizeof(kfc))) - return -EFAULT; - rc = cca_findcard(kfc.seckey.seckey, - &kfc.cardnr, &kfc.domain, 1); - DEBUG_DBG("%s cca_findcard()=%d\n", __func__, rc); - if (rc < 0) - break; - if (copy_to_user(ufc, &kfc, sizeof(kfc))) - return -EFAULT; + case PKEY_FINDCARD: + rc = pkey_ioctl_findcard((struct pkey_findcard __user *)arg); break; - } - case PKEY_SKEY2PKEY: { - struct pkey_skey2pkey __user *usp = (void __user *)arg; - struct pkey_skey2pkey ksp; - - if (copy_from_user(&ksp, usp, sizeof(ksp))) - return -EFAULT; - ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(usp, &ksp, sizeof(ksp))) - return -EFAULT; + case PKEY_SKEY2PKEY: + rc = pkey_ioctl_skey2pkey((struct pkey_skey2pkey __user *)arg); break; - } - case PKEY_VERIFYKEY: { - struct pkey_verifykey __user *uvk = (void __user *)arg; - struct pkey_verifykey kvk; - - if (copy_from_user(&kvk, uvk, sizeof(kvk))) - return -EFAULT; - rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, - &kvk.keysize, &kvk.attributes); - DEBUG_DBG("%s pkey_verifykey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(uvk, &kvk, sizeof(kvk))) - return -EFAULT; + case PKEY_VERIFYKEY: + rc = pkey_ioctl_verifykey((struct pkey_verifykey __user *)arg); break; - } - case PKEY_GENPROTK: { - struct pkey_genprotk __user *ugp = (void __user *)arg; - struct pkey_genprotk kgp; - - if (copy_from_user(&kgp, ugp, sizeof(kgp))) - return -EFAULT; - kgp.protkey.len = sizeof(kgp.protkey.protkey); - rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, - &kgp.protkey.len, &kgp.protkey.type); - DEBUG_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); - if (rc) - break; - if (copy_to_user(ugp, &kgp, sizeof(kgp))) - return -EFAULT; + case PKEY_GENPROTK: + rc = pkey_ioctl_genprotk((struct pkey_genprotk __user *)arg); break; - } - case PKEY_VERIFYPROTK: { - struct pkey_verifyprotk __user *uvp = (void __user *)arg; - struct pkey_verifyprotk kvp; - - if (copy_from_user(&kvp, uvp, sizeof(kvp))) - return -EFAULT; - rc = pkey_verifyprotkey(kvp.protkey.protkey, - kvp.protkey.len, kvp.protkey.type); - DEBUG_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); + case PKEY_VERIFYPROTK: + rc = pkey_ioctl_verifyprotk((struct pkey_verifyprotk __user *)arg); break; - } - case PKEY_KBLOB2PROTK: { - struct pkey_kblob2pkey __user *utp = (void __user *)arg; - struct pkey_kblob2pkey ktp; - u8 *kkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) - return PTR_ERR(kkey); - ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, - &ktp.protkey.len, &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); - memzero_explicit(kkey, ktp.keylen); - kfree(kkey); - if (rc) - break; - if (copy_to_user(utp, &ktp, sizeof(ktp))) - return -EFAULT; + case PKEY_KBLOB2PROTK: + rc = pkey_ioctl_kblob2protk((struct pkey_kblob2pkey __user *)arg); break; - } - case PKEY_GENSECK2: { - struct pkey_genseck2 __user *ugs = (void __user *)arg; - size_t klen = KEYBLOBBUFSIZE; - struct pkey_genseck2 kgs; - struct pkey_apqn *apqns; - u8 *kkey; - - if (copy_from_user(&kgs, ugs, sizeof(kgs))) - return -EFAULT; - apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = kzalloc(klen, GFP_KERNEL); - if (!kkey) { - kfree(apqns); - return -ENOMEM; - } - rc = pkey_genseckey2(apqns, kgs.apqn_entries, - kgs.type, kgs.size, kgs.keygenflags, - kkey, &klen); - DEBUG_DBG("%s pkey_genseckey2()=%d\n", __func__, rc); - kfree(apqns); - if (rc) { - kfree(kkey); - break; - } - if (kgs.key) { - if (kgs.keylen < klen) { - kfree(kkey); - return -EINVAL; - } - if (copy_to_user(kgs.key, kkey, klen)) { - kfree(kkey); - return -EFAULT; - } - } - kgs.keylen = klen; - if (copy_to_user(ugs, &kgs, sizeof(kgs))) - rc = -EFAULT; - kfree(kkey); + case PKEY_GENSECK2: + rc = pkey_ioctl_genseck2((struct pkey_genseck2 __user *)arg); break; - } - case PKEY_CLR2SECK2: { - struct pkey_clr2seck2 __user *ucs = (void __user *)arg; - size_t klen = KEYBLOBBUFSIZE; - struct pkey_clr2seck2 kcs; - struct pkey_apqn *apqns; - u8 *kkey; - - if (copy_from_user(&kcs, ucs, sizeof(kcs))) - return -EFAULT; - apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = kzalloc(klen, GFP_KERNEL); - if (!kkey) { - kfree(apqns); - return -ENOMEM; - } - rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, - kcs.type, kcs.size, kcs.keygenflags, - kcs.clrkey.clrkey, kkey, &klen); - DEBUG_DBG("%s pkey_clr2seckey2()=%d\n", __func__, rc); - kfree(apqns); - if (rc) { - kfree(kkey); - break; - } - if (kcs.key) { - if (kcs.keylen < klen) { - kfree(kkey); - return -EINVAL; - } - if (copy_to_user(kcs.key, kkey, klen)) { - kfree(kkey); - return -EFAULT; - } - } - kcs.keylen = klen; - if (copy_to_user(ucs, &kcs, sizeof(kcs))) - rc = -EFAULT; - memzero_explicit(&kcs, sizeof(kcs)); - kfree(kkey); + case PKEY_CLR2SECK2: + rc = pkey_ioctl_clr2seck2((struct pkey_clr2seck2 __user *)arg); break; - } - case PKEY_VERIFYKEY2: { - struct pkey_verifykey2 __user *uvk = (void __user *)arg; - struct pkey_verifykey2 kvk; - u8 *kkey; - - if (copy_from_user(&kvk, uvk, sizeof(kvk))) - return -EFAULT; - kkey = _copy_key_from_user(kvk.key, kvk.keylen); - if (IS_ERR(kkey)) - return PTR_ERR(kkey); - rc = pkey_verifykey2(kkey, kvk.keylen, - &kvk.cardnr, &kvk.domain, - &kvk.type, &kvk.size, &kvk.flags); - DEBUG_DBG("%s pkey_verifykey2()=%d\n", __func__, rc); - kfree(kkey); - if (rc) - break; - if (copy_to_user(uvk, &kvk, sizeof(kvk))) - return -EFAULT; + case PKEY_VERIFYKEY2: + rc = pkey_ioctl_verifykey2((struct pkey_verifykey2 __user *)arg); break; - } - case PKEY_KBLOB2PROTK2: { - struct pkey_kblob2pkey2 __user *utp = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_kblob2pkey2 ktp; - u8 *kkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey2(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - ktp.protkey.protkey, &ktp.protkey.len, - &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); - kfree(apqns); - memzero_explicit(kkey, ktp.keylen); - kfree(kkey); - if (rc) - break; - if (copy_to_user(utp, &ktp, sizeof(ktp))) - return -EFAULT; + case PKEY_KBLOB2PROTK2: + rc = pkey_ioctl_kblob2protk2((struct pkey_kblob2pkey2 __user *)arg); break; - } - case PKEY_APQNS4K: { - struct pkey_apqns4key __user *uak = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_apqns4key kak; - size_t nr_apqns, len; - u8 *kkey; - - if (copy_from_user(&kak, uak, sizeof(kak))) - return -EFAULT; - nr_apqns = kak.apqn_entries; - if (nr_apqns) { - apqns = kmalloc_array(nr_apqns, - sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!apqns) - return -ENOMEM; - } - kkey = _copy_key_from_user(kak.key, kak.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, - apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4key()=%d\n", __func__, rc); - kfree(kkey); - if (rc && rc != -ENOSPC) { - kfree(apqns); - break; - } - if (!rc && kak.apqns) { - if (nr_apqns > kak.apqn_entries) { - kfree(apqns); - return -EINVAL; - } - len = nr_apqns * sizeof(struct pkey_apqn); - if (len) { - if (copy_to_user(kak.apqns, apqns, len)) { - kfree(apqns); - return -EFAULT; - } - } - } - kak.apqn_entries = nr_apqns; - if (copy_to_user(uak, &kak, sizeof(kak))) - rc = -EFAULT; - kfree(apqns); + case PKEY_APQNS4K: + rc = pkey_ioctl_apqns4k((struct pkey_apqns4key __user *)arg); break; - } - case PKEY_APQNS4KT: { - struct pkey_apqns4keytype __user *uat = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_apqns4keytype kat; - size_t nr_apqns, len; - - if (copy_from_user(&kat, uat, sizeof(kat))) - return -EFAULT; - nr_apqns = kat.apqn_entries; - if (nr_apqns) { - apqns = kmalloc_array(nr_apqns, - sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!apqns) - return -ENOMEM; - } - rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, - kat.flags, apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4keytype()=%d\n", __func__, rc); - if (rc && rc != -ENOSPC) { - kfree(apqns); - break; - } - if (!rc && kat.apqns) { - if (nr_apqns > kat.apqn_entries) { - kfree(apqns); - return -EINVAL; - } - len = nr_apqns * sizeof(struct pkey_apqn); - if (len) { - if (copy_to_user(kat.apqns, apqns, len)) { - kfree(apqns); - return -EFAULT; - } - } - } - kat.apqn_entries = nr_apqns; - if (copy_to_user(uat, &kat, sizeof(kat))) - rc = -EFAULT; - kfree(apqns); + case PKEY_APQNS4KT: + rc = pkey_ioctl_apqns4kt((struct pkey_apqns4keytype __user *)arg); break; - } - case PKEY_KBLOB2PROTK3: { - struct pkey_kblob2pkey3 __user *utp = (void __user *)arg; - u32 protkeylen = PROTKEYBLOBBUFSIZE; - struct pkey_apqn *apqns = NULL; - struct pkey_kblob2pkey3 ktp; - u8 *kkey, *protkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - protkey = kmalloc(protkeylen, GFP_KERNEL); - if (!protkey) { - kfree(apqns); - kfree(kkey); - return -ENOMEM; - } - rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - protkey, &protkeylen, &ktp.pkeytype); - DEBUG_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); - kfree(apqns); - memzero_explicit(kkey, ktp.keylen); - kfree(kkey); - if (rc) { - kfree(protkey); - break; - } - if (ktp.pkey && ktp.pkeylen) { - if (protkeylen > ktp.pkeylen) { - kfree(protkey); - return -EINVAL; - } - if (copy_to_user(ktp.pkey, protkey, protkeylen)) { - kfree(protkey); - return -EFAULT; - } - } - kfree(protkey); - ktp.pkeylen = protkeylen; - if (copy_to_user(utp, &ktp, sizeof(ktp))) - return -EFAULT; + case PKEY_KBLOB2PROTK3: + rc = pkey_ioctl_kblob2protk3((struct pkey_kblob2pkey3 __user *)arg); break; - } default: /* unknown/unsupported ioctl cmd */ return -ENOTTY; @@ -1790,499 +772,12 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } /* - * Sysfs and file io operations - */ - -/* - * Sysfs attribute read function for all protected key binary attributes. - * The implementation can not deal with partial reads, because a new random - * protected key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - */ -static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, - loff_t off, size_t count) -{ - struct protaeskeytoken protkeytoken; - struct pkey_protkey protkey; - int rc; - - if (off != 0 || count < sizeof(protkeytoken)) - return -EINVAL; - if (is_xts) - if (count < 2 * sizeof(protkeytoken)) - return -EINVAL; - - memset(&protkeytoken, 0, sizeof(protkeytoken)); - protkeytoken.type = TOKTYPE_NON_CCA; - protkeytoken.version = TOKVER_PROTECTED_KEY; - protkeytoken.keytype = keytype; - - protkey.len = sizeof(protkey.protkey); - rc = pkey_genprotkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, &protkey.type); - if (rc) - return rc; - - protkeytoken.len = protkey.len; - memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); - - memcpy(buf, &protkeytoken, sizeof(protkeytoken)); - - if (is_xts) { - /* xts needs a second protected key, reuse protkey struct */ - protkey.len = sizeof(protkey.protkey); - rc = pkey_genprotkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, &protkey.type); - if (rc) - return rc; - - protkeytoken.len = protkey.len; - memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); - - memcpy(buf + sizeof(protkeytoken), &protkeytoken, - sizeof(protkeytoken)); - - return 2 * sizeof(protkeytoken); - } - - return sizeof(protkeytoken); -} - -static ssize_t protkey_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, - off, count); -} - -static ssize_t protkey_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, - off, count); -} - -static ssize_t protkey_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, - off, count); -} - -static ssize_t protkey_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, - off, count); -} - -static ssize_t protkey_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken)); - -static struct bin_attribute *protkey_attrs[] = { - &bin_attr_protkey_aes_128, - &bin_attr_protkey_aes_192, - &bin_attr_protkey_aes_256, - &bin_attr_protkey_aes_128_xts, - &bin_attr_protkey_aes_256_xts, - NULL -}; - -static struct attribute_group protkey_attr_group = { - .name = "protkey", - .bin_attrs = protkey_attrs, -}; - -/* - * Sysfs attribute read function for all secure key ccadata binary attributes. - * The implementation can not deal with partial reads, because a new random - * protected key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + * File io operations */ -static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, - loff_t off, size_t count) -{ - struct pkey_seckey *seckey = (struct pkey_seckey *)buf; - int rc; - - if (off != 0 || count < sizeof(struct secaeskeytoken)) - return -EINVAL; - if (is_xts) - if (count < 2 * sizeof(struct secaeskeytoken)) - return -EINVAL; - - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); - if (rc) - return rc; - - if (is_xts) { - seckey++; - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); - if (rc) - return rc; - - return 2 * sizeof(struct secaeskeytoken); - } - - return sizeof(struct secaeskeytoken); -} - -static ssize_t ccadata_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, - off, count); -} - -static ssize_t ccadata_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, - off, count); -} - -static ssize_t ccadata_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, - off, count); -} - -static ssize_t ccadata_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, - off, count); -} - -static ssize_t ccadata_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ccadata_aes_128, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_192, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_256, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_128_xts, 2 * sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_256_xts, 2 * sizeof(struct secaeskeytoken)); - -static struct bin_attribute *ccadata_attrs[] = { - &bin_attr_ccadata_aes_128, - &bin_attr_ccadata_aes_192, - &bin_attr_ccadata_aes_256, - &bin_attr_ccadata_aes_128_xts, - &bin_attr_ccadata_aes_256_xts, - NULL -}; - -static struct attribute_group ccadata_attr_group = { - .name = "ccadata", - .bin_attrs = ccadata_attrs, -}; - -#define CCACIPHERTOKENSIZE (sizeof(struct cipherkeytoken) + 80) - -/* - * Sysfs attribute read function for all secure key ccacipher binary attributes. - * The implementation can not deal with partial reads, because a new random - * secure key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - */ -static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, - bool is_xts, char *buf, loff_t off, - size_t count) -{ - size_t keysize = CCACIPHERTOKENSIZE; - u32 nr_apqns, *apqns = NULL; - int i, rc, card, dom; - - if (off != 0 || count < CCACIPHERTOKENSIZE) - return -EINVAL; - if (is_xts) - if (count < 2 * CCACIPHERTOKENSIZE) - return -EINVAL; - - /* build a list of apqns able to generate an cipher key */ - rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX6, 0, 0, 0, 0); - if (rc) - return rc; - - memset(buf, 0, is_xts ? 2 * keysize : keysize); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - break; - } - if (rc) - return rc; - - if (is_xts) { - keysize = CCACIPHERTOKENSIZE; - buf += CCACIPHERTOKENSIZE; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - return 2 * CCACIPHERTOKENSIZE; - } - - return CCACIPHERTOKENSIZE; -} - -static ssize_t ccacipher_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_192, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, true, buf, - off, count); -} - -static ssize_t ccacipher_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ccacipher_aes_128, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_192, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_256, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_128_xts, 2 * CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_256_xts, 2 * CCACIPHERTOKENSIZE); - -static struct bin_attribute *ccacipher_attrs[] = { - &bin_attr_ccacipher_aes_128, - &bin_attr_ccacipher_aes_192, - &bin_attr_ccacipher_aes_256, - &bin_attr_ccacipher_aes_128_xts, - &bin_attr_ccacipher_aes_256_xts, - NULL -}; - -static struct attribute_group ccacipher_attr_group = { - .name = "ccacipher", - .bin_attrs = ccacipher_attrs, -}; - -/* - * Sysfs attribute read function for all ep11 aes key binary attributes. - * The implementation can not deal with partial reads, because a new random - * secure key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - * This function and the sysfs attributes using it provide EP11 key blobs - * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently - * 336 bytes. - */ -static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, - bool is_xts, char *buf, loff_t off, - size_t count) -{ - size_t keysize = MAXEP11AESKEYBLOBSIZE; - u32 nr_apqns, *apqns = NULL; - int i, rc, card, dom; - - if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) - return -EINVAL; - if (is_xts) - if (count < 2 * MAXEP11AESKEYBLOBSIZE) - return -EINVAL; - - /* build a list of apqns able to generate an cipher key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - NULL); - if (rc) - return rc; - - memset(buf, 0, is_xts ? 2 * keysize : keysize); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - break; - } - if (rc) - return rc; - - if (is_xts) { - keysize = MAXEP11AESKEYBLOBSIZE; - buf += MAXEP11AESKEYBLOBSIZE; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - return 2 * MAXEP11AESKEYBLOBSIZE; - } - - return MAXEP11AESKEYBLOBSIZE; -} - -static ssize_t ep11_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf, - off, count); -} - -static ssize_t ep11_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf, - off, count); -} - -static ssize_t ep11_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf, - off, count); -} - -static ssize_t ep11_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf, - off, count); -} - -static ssize_t ep11_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE); - -static struct bin_attribute *ep11_attrs[] = { - &bin_attr_ep11_aes_128, - &bin_attr_ep11_aes_192, - &bin_attr_ep11_aes_256, - &bin_attr_ep11_aes_128_xts, - &bin_attr_ep11_aes_256_xts, - NULL -}; - -static struct attribute_group ep11_attr_group = { - .name = "ep11", - .bin_attrs = ep11_attrs, -}; - -static const struct attribute_group *pkey_attr_groups[] = { - &protkey_attr_group, - &ccadata_attr_group, - &ccacipher_attr_group, - &ep11_attr_group, - NULL, -}; static const struct file_operations pkey_fops = { .owner = THIS_MODULE, .open = nonseekable_open, - .llseek = no_llseek, .unlocked_ioctl = pkey_unlocked_ioctl, }; @@ -2294,43 +789,13 @@ static struct miscdevice pkey_dev = { .groups = pkey_attr_groups, }; -/* - * Module init - */ -static int __init pkey_init(void) +int __init pkey_api_init(void) { - cpacf_mask_t func_mask; - - /* - * The pckmo instruction should be available - even if we don't - * actually invoke it. This instruction comes with MSA 3 which - * is also the minimum level for the kmc instructions which - * are able to work with protected keys. - */ - if (!cpacf_query(CPACF_PCKMO, &func_mask)) - return -ENODEV; - - /* check for kmc instructions available */ - if (!cpacf_query(CPACF_KMC, &func_mask)) - return -ENODEV; - if (!cpacf_test_func(&func_mask, CPACF_KMC_PAES_128) || - !cpacf_test_func(&func_mask, CPACF_KMC_PAES_192) || - !cpacf_test_func(&func_mask, CPACF_KMC_PAES_256)) - return -ENODEV; - - pkey_debug_init(); - + /* register as a misc device */ return misc_register(&pkey_dev); } -/* - * Module exit - */ -static void __exit pkey_exit(void) +void __exit pkey_api_exit(void) { misc_deregister(&pkey_dev); - pkey_debug_exit(); } - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init); -module_exit(pkey_exit); diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c new file mode 100644 index 000000000000..d60cd987c16d --- /dev/null +++ b/drivers/s390/crypto/pkey_base.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey base: debug feature, pkey handler registry + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/cpufeature.h> +#include <linux/export.h> +#include <linux/init.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/rculist.h> + +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key base and api"); + +/* + * pkey debug feature + */ +debug_info_t *pkey_dbf_info; +EXPORT_SYMBOL(pkey_dbf_info); + +/* + * pkey handler registry + */ + +static DEFINE_SPINLOCK(handler_list_write_lock); +static LIST_HEAD(handler_list); + +int pkey_handler_register(struct pkey_handler *handler) +{ + const struct pkey_handler *h; + + if (!handler || + !handler->is_supported_key || + !handler->is_supported_keytype) + return -EINVAL; + + if (!try_module_get(handler->module)) + return -ENXIO; + + spin_lock(&handler_list_write_lock); + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h == handler) { + rcu_read_unlock(); + spin_unlock(&handler_list_write_lock); + module_put(handler->module); + return -EEXIST; + } + } + rcu_read_unlock(); + + list_add_rcu(&handler->list, &handler_list); + spin_unlock(&handler_list_write_lock); + synchronize_rcu(); + + module_put(handler->module); + + PKEY_DBF_INFO("%s pkey handler '%s' registered\n", __func__, + handler->name ?: "<no name>"); + + return 0; +} +EXPORT_SYMBOL(pkey_handler_register); + +int pkey_handler_unregister(struct pkey_handler *handler) +{ + spin_lock(&handler_list_write_lock); + list_del_rcu(&handler->list); + INIT_LIST_HEAD_RCU(&handler->list); + spin_unlock(&handler_list_write_lock); + synchronize_rcu(); + + PKEY_DBF_INFO("%s pkey handler '%s' unregistered\n", __func__, + handler->name ?: "<no name>"); + + return 0; +} +EXPORT_SYMBOL(pkey_handler_unregister); + +/* + * Handler invocation functions. + */ + +const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen) +{ + const struct pkey_handler *h; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->is_supported_key(key, keylen)) { + rcu_read_unlock(); + return h; + } + module_put(h->module); + } + rcu_read_unlock(); + + return NULL; +} +EXPORT_SYMBOL(pkey_handler_get_keybased); + +const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt) +{ + const struct pkey_handler *h; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->is_supported_keytype(kt)) { + rcu_read_unlock(); + return h; + } + module_put(h->module); + } + rcu_read_unlock(); + + return NULL; +} +EXPORT_SYMBOL(pkey_handler_get_keytypebased); + +void pkey_handler_put(const struct pkey_handler *handler) +{ + const struct pkey_handler *h; + + if (!handler) + return; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h == handler) { + module_put(h->module); + break; + } + } + rcu_read_unlock(); +} +EXPORT_SYMBOL(pkey_handler_put); + +int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->key_to_protkey) { + rc = h->key_to_protkey(apqns, nr_apqns, key, keylen, + protkey, protkeylen, + protkeytype, xflags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_key_to_protkey); + +/* + * This handler invocation is special as there may be more than + * one handler providing support for the very same key (type). + * And the handler may not respond true on is_supported_key(), + * so simple try and check return value here. + */ +int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype, u32 xflags) +{ + const struct pkey_handler *h, *htmp[10]; + int i, n = 0, rc = -ENODEV; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->slowpath_key_to_protkey && n < ARRAY_SIZE(htmp)) + htmp[n++] = h; + else + module_put(h->module); + } + rcu_read_unlock(); + + for (i = 0; i < n; i++) { + h = htmp[i]; + if (rc) + rc = h->slowpath_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype, xflags); + module_put(h->module); + } + + return rc; +} +EXPORT_SYMBOL(pkey_handler_slowpath_key_to_protkey); + +int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->gen_key) { + rc = h->gen_key(apqns, nr_apqns, keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo, xflags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_gen_key); + +int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, + u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->clr_to_key) { + rc = h->clr_to_key(apqns, nr_apqns, keytype, keysubtype, + keybitsize, flags, clrkey, clrkeylen, + keybuf, keybuflen, keyinfo, xflags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_clr_to_key); + +int pkey_handler_verify_key(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags, + u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->verify_key) { + rc = h->verify_key(key, keylen, card, dom, + keytype, keybitsize, flags, xflags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_verify_key); + +int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->apqns_for_key) + rc = h->apqns_for_key(key, keylen, flags, apqns, nr_apqns, + xflags); + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_apqns_for_key); + +int pkey_handler_apqns_for_keytype(enum pkey_key_type keysubtype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->apqns_for_keytype) { + rc = h->apqns_for_keytype(keysubtype, + cur_mkvp, alt_mkvp, flags, + apqns, nr_apqns, xflags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_apqns_for_keytype); + +void pkey_handler_request_modules(void) +{ +#ifdef CONFIG_MODULES + static const char * const pkey_handler_modules[] = { +#if IS_MODULE(CONFIG_PKEY_CCA) + "pkey_cca", +#endif +#if IS_MODULE(CONFIG_PKEY_EP11) + "pkey_ep11", +#endif +#if IS_MODULE(CONFIG_PKEY_PCKMO) + "pkey_pckmo", +#endif +#if IS_MODULE(CONFIG_PKEY_UV) + "pkey_uv", +#endif + }; + int i; + + for (i = 0; i < ARRAY_SIZE(pkey_handler_modules); i++) { + const struct pkey_handler *h; + bool found = false; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h->module && + !strcmp(h->module->name, pkey_handler_modules[i])) { + found = true; + break; + } + } + rcu_read_unlock(); + if (!found) { + pr_debug("request_module(%s)\n", pkey_handler_modules[i]); + request_module(pkey_handler_modules[i]); + } + } +#endif +} +EXPORT_SYMBOL(pkey_handler_request_modules); + +/* + * Module init + */ +static int __init pkey_init(void) +{ + int rc; + + /* init debug feature */ + pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); + debug_register_view(pkey_dbf_info, &debug_sprintf_view); + debug_set_level(pkey_dbf_info, 4); + + /* the handler registry does not need any init */ + + rc = pkey_api_init(); + if (rc) + debug_unregister(pkey_dbf_info); + + return rc; +} + +/* + * Module exit + */ +static void __exit pkey_exit(void) +{ + pkey_api_exit(); +} + +module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init); +module_exit(pkey_exit); diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h new file mode 100644 index 000000000000..9cdb3e74477f --- /dev/null +++ b/drivers/s390/crypto/pkey_base.h @@ -0,0 +1,240 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright IBM Corp. 2024 + * + * Pkey base: debug feature, defines and structs + * common to all pkey code. + */ + +#ifndef _PKEY_BASE_H_ +#define _PKEY_BASE_H_ + +#include <linux/types.h> +#include <asm/debug.h> +#include <asm/pkey.h> + +/* + * pkey debug feature + */ + +extern debug_info_t *pkey_dbf_info; + +#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) +#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) +#define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) + +/* + * common defines and common structs + */ + +#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ +#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header)) +#define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ +#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ +#define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ + +/* inside view of a generic protected key token */ +struct protkeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* should be 0x01 for protected key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ + u32 len; /* bytes actually stored in protkey[] */ + u8 protkey[]; /* the protected key blob */ +} __packed; + +/* inside view of a protected AES key token */ +struct protaeskeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* should be 0x01 for protected key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ + u32 len; /* bytes actually stored in protkey[] */ + u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ +} __packed; + +/* inside view of a clear key token (type 0x00 version 0x02) */ +struct clearkeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* 0x02 for clear key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE_* values */ + u32 len; /* bytes actually stored in clearkey[] */ + u8 clearkey[]; /* clear key value */ +} __packed; + +/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */ +static inline u32 pkey_keytype_aes_to_size(u32 keytype) +{ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + return 16; + case PKEY_KEYTYPE_AES_192: + return 24; + case PKEY_KEYTYPE_AES_256: + return 32; + default: + return 0; + } +} + +/* helper function which translates AES key bit size into PKEY_KEYTYPE_AES_* */ +static inline u32 pkey_aes_bitsize_to_keytype(u32 keybitsize) +{ + switch (keybitsize) { + case 128: + return PKEY_KEYTYPE_AES_128; + case 192: + return PKEY_KEYTYPE_AES_192; + case 256: + return PKEY_KEYTYPE_AES_256; + default: + return 0; + } +} + +/* + * helper function which translates the PKEY_KEYTYPE_* + * to the protected key size minus the WK VP length + */ +static inline u32 pkey_keytype_to_size(u32 keytype) +{ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + return 16; + case PKEY_KEYTYPE_AES_192: + return 24; + case PKEY_KEYTYPE_AES_256: + return 32; + case PKEY_KEYTYPE_ECC_P256: + return 32; + case PKEY_KEYTYPE_ECC_P384: + return 48; + case PKEY_KEYTYPE_ECC_P521: + return 80; + case PKEY_KEYTYPE_ECC_ED25519: + return 32; + case PKEY_KEYTYPE_ECC_ED448: + return 54; + case PKEY_KEYTYPE_AES_XTS_128: + return 32; + case PKEY_KEYTYPE_AES_XTS_256: + return 64; + case PKEY_KEYTYPE_HMAC_512: + return 64; + case PKEY_KEYTYPE_HMAC_1024: + return 128; + default: + return 0; + } +} + +/* + * pkey_api.c: + */ +int __init pkey_api_init(void); +void __exit pkey_api_exit(void); + +/* + * pkey_sysfs.c: + */ + +extern const struct attribute_group *pkey_attr_groups[]; + +/* + * pkey handler registry + */ + +struct pkey_handler { + struct module *module; + const char *name; + /* + * is_supported_key() and is_supported_keytype() are called + * within an rcu_read_lock() scope and thus must not sleep! + */ + bool (*is_supported_key)(const u8 *key, u32 keylen); + bool (*is_supported_keytype)(enum pkey_key_type); + int (*key_to_protkey)(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags); + int (*slowpath_key_to_protkey)(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype, u32 xflags); + int (*gen_key)(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags); + int (*clr_to_key)(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags); + int (*verify_key)(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags, + u32 xflags); + int (*apqns_for_key)(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags); + int (*apqns_for_keytype)(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags); + /* used internal by pkey base */ + struct list_head list; +}; + +int pkey_handler_register(struct pkey_handler *handler); +int pkey_handler_unregister(struct pkey_handler *handler); + +/* + * invocation function for the registered pkey handlers + */ + +const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen); +const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt); +void pkey_handler_put(const struct pkey_handler *handler); + +int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags); +int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype, u32 xflags); +int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags); +int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, + u32 xflags); +int pkey_handler_verify_key(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags, + u32 xflags); +int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags); +int pkey_handler_apqns_for_keytype(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 xflags); + +/* + * Unconditional try to load all handler modules + */ +void pkey_handler_request_modules(void); + +#endif /* _PKEY_BASE_H_ */ diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c new file mode 100644 index 000000000000..d4550d8d8eea --- /dev/null +++ b/drivers/s390/crypto/pkey_cca.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey cca specific code + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/cpufeature.h> + +#include "zcrypt_ccamisc.h" +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key CCA handler"); + +#if IS_MODULE(CONFIG_PKEY_CCA) +static struct ap_device_id pkey_cca_card_ids[] = { + { .dev_type = AP_DEVICE_TYPE_CEX4 }, + { .dev_type = AP_DEVICE_TYPE_CEX5 }, + { .dev_type = AP_DEVICE_TYPE_CEX6 }, + { .dev_type = AP_DEVICE_TYPE_CEX7 }, + { .dev_type = AP_DEVICE_TYPE_CEX8 }, + { /* end of list */ }, +}; +MODULE_DEVICE_TABLE(ap, pkey_cca_card_ids); +#endif + +/* + * Check key blob for known and supported CCA key. + */ +static bool is_cca_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_CCA_INTERNAL: + switch (hdr->version) { + case TOKVER_CCA_AES: + case TOKVER_CCA_VLSC: + return true; + default: + return false; + } + case TOKTYPE_CCA_INTERNAL_PKA: + return true; + default: + return false; + } +} + +static bool is_cca_keytype(enum pkey_key_type key_type) +{ + switch (key_type) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + case PKEY_TYPE_CCA_ECC: + return true; + default: + return false; + } +} + +static int cca_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (!flags) + flags = PKEY_FLAGS_MATCH_CUR_MKVP | PKEY_FLAGS_MATCH_ALT_MKVP; + + if (keylen < sizeof(struct keytoken_header)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL) { + u64 cur_mkvp = 0, old_mkvp = 0; + int minhwtype = ZCRYPT_CEX3C; + + if (hdr->version == TOKVER_CCA_AES) { + struct secaeskeytoken *t = (struct secaeskeytoken *)key; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp; + } else if (hdr->version == TOKVER_CCA_VLSC) { + struct cipherkeytoken *t = (struct cipherkeytoken *)key; + + minhwtype = ZCRYPT_CEX6; + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp0; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp0; + } else { + /* unknown CCA internal token type */ + return -EINVAL; + } + rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, xflags); + if (rc) + goto out; + + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; + u64 cur_mkvp = 0, old_mkvp = 0; + + if (t->secid == 0x20) { + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp; + } else { + /* unknown CCA internal 2 token type */ + return -EINVAL; + } + rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, xflags); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int cca_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, + u32 pflags) +{ + u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + zcrypt_wait_api_operational(); + + if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { + u64 cur_mkvp = 0, old_mkvp = 0; + int minhwtype = ZCRYPT_CEX3C; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = *((u64 *)cur_mkvp); + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = *((u64 *)alt_mkvp); + if (ktype == PKEY_TYPE_CCA_CIPHER) + minhwtype = ZCRYPT_CEX6; + rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, xflags); + if (rc) + goto out; + + } else if (ktype == PKEY_TYPE_CCA_ECC) { + u64 cur_mkvp = 0, old_mkvp = 0; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = *((u64 *)cur_mkvp); + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = *((u64 *)alt_mkvp); + rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, xflags); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported key type %d", + __func__, (int)ktype); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int cca_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + u32 xflags; + int i, rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + /* CCA AES data key */ + if (keylen < sizeof(struct secaeskeytoken)) + return -EINVAL; + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + /* CCA AES cipher key */ + if (keylen < hdr->len) + return -EINVAL; + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + /* CCA ECC (private) key */ + if (keylen < sizeof(struct eccprivkeytoken)) + return -EINVAL; + if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = cca_apqns4key(key, keylen, 0, _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + rc = cca_sec2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype, xflags); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + rc = cca_cipher2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype, xflags); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + rc = cca_ecc2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype, xflags); + } else { + rc = -EINVAL; + break; + } + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate CCA secure key. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int cca_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags) +{ + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + int i, len, rc; + u32 xflags; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = cca_apqns4type(subtype, NULL, NULL, 0, + _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_gencipherkey(apqns[i].card, apqns[i].domain, + keybitsize, flags, + keybuf, keybuflen, xflags); + } else { + /* PKEY_TYPE_CCA_DATA */ + rc = cca_genseckey(apqns[i].card, apqns[i].domain, + keybitsize, keybuf, xflags); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate CCA secure key with given clear key value. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int cca_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags) +{ + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + int i, len, rc; + u32 xflags; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = cca_apqns4type(subtype, NULL, NULL, 0, + _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_clr2cipherkey(apqns[i].card, apqns[i].domain, + keybitsize, flags, clrkey, + keybuf, keybuflen, xflags); + } else { + /* PKEY_TYPE_CCA_DATA */ + rc = cca_clr2seckey(apqns[i].card, apqns[i].domain, + keybitsize, clrkey, keybuf, xflags); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int cca_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags, u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 apqns[MAXAPQNSINLIST], nr_apqns = ARRAY_SIZE(apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + struct secaeskeytoken *t = (struct secaeskeytoken *)key; + + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_DATA; + *keybitsize = t->bitsize; + rc = cca_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + t->mkvp, 0, xflags); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + nr_apqns = ARRAY_SIZE(apqns); + rc = cca_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + 0, t->mkvp, xflags); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + struct cipherkeytoken *t = (struct cipherkeytoken *)key; + + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_CIPHER; + *keybitsize = PKEY_SIZE_UNKNOWN; + if (!t->plfver && t->wpllen == 512) + *keybitsize = PKEY_SIZE_AES_128; + else if (!t->plfver && t->wpllen == 576) + *keybitsize = PKEY_SIZE_AES_192; + else if (!t->plfver && t->wpllen == 640) + *keybitsize = PKEY_SIZE_AES_256; + rc = cca_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + t->mkvp0, 0, xflags); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + nr_apqns = ARRAY_SIZE(apqns); + rc = cca_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + 0, t->mkvp0, xflags); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * This function provides an alternate but usually slow way + * to convert a 'clear key token' with AES key material into + * a protected key. This is done via an intermediate step + * which creates a CCA AES DATA secure key first and then + * derives the protected key from this secure key. + */ +static int cca_slowpath_key2protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype, u32 pflags) +{ + const struct keytoken_header *hdr = (const struct keytoken_header *)key; + const struct clearkeytoken *t = (const struct clearkeytoken *)key; + u8 tmpbuf[SECKEYBLOBSIZE]; /* 64 bytes */ + u32 tmplen, keysize = 0; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_CLEAR_KEY) + keysize = pkey_keytype_aes_to_size(t->keytype); + if (!keysize || t->len != keysize) + return -EINVAL; + + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + tmplen = SECKEYBLOBSIZE; + rc = cca_clr2key(NULL, 0, t->keytype, PKEY_TYPE_CCA_DATA, + 8 * keysize, 0, t->clearkey, t->len, + tmpbuf, &tmplen, NULL, pflags); + pr_debug("cca_clr2key()=%d\n", rc); + if (rc) + continue; + rc = cca_key2protkey(NULL, 0, tmpbuf, tmplen, + protkey, protkeylen, protkeytype, pflags); + pr_debug("cca_key2protkey()=%d\n", rc); + } + + pr_debug("rc=%d\n", rc); + return rc; +} + +static struct pkey_handler cca_handler = { + .module = THIS_MODULE, + .name = "PKEY CCA handler", + .is_supported_key = is_cca_key, + .is_supported_keytype = is_cca_keytype, + .key_to_protkey = cca_key2protkey, + .slowpath_key_to_protkey = cca_slowpath_key2protkey, + .gen_key = cca_gen_key, + .clr_to_key = cca_clr2key, + .verify_key = cca_verifykey, + .apqns_for_key = cca_apqns4key, + .apqns_for_keytype = cca_apqns4type, +}; + +/* + * Module init + */ +static int __init pkey_cca_init(void) +{ + /* register this module as pkey handler for all the cca stuff */ + return pkey_handler_register(&cca_handler); +} + +/* + * Module exit + */ +static void __exit pkey_cca_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&cca_handler); +} + +module_init(pkey_cca_init); +module_exit(pkey_cca_exit); diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c new file mode 100644 index 000000000000..654eed20d0d9 --- /dev/null +++ b/drivers/s390/crypto/pkey_ep11.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey ep11 specific code + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/cpufeature.h> + +#include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key EP11 handler"); + +#if IS_MODULE(CONFIG_PKEY_EP11) +static struct ap_device_id pkey_ep11_card_ids[] = { + { .dev_type = AP_DEVICE_TYPE_CEX4 }, + { .dev_type = AP_DEVICE_TYPE_CEX5 }, + { .dev_type = AP_DEVICE_TYPE_CEX6 }, + { .dev_type = AP_DEVICE_TYPE_CEX7 }, + { .dev_type = AP_DEVICE_TYPE_CEX8 }, + { /* end of list */ }, +}; +MODULE_DEVICE_TABLE(ap, pkey_ep11_card_ids); +#endif + +/* + * Check key blob for known and supported EP11 key. + */ +static bool is_ep11_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_NON_CCA: + switch (hdr->version) { + case TOKVER_EP11_AES: + case TOKVER_EP11_AES_WITH_HEADER: + case TOKVER_EP11_ECC_WITH_HEADER: + return true; + default: + return false; + } + default: + return false; + } +} + +static bool is_ep11_keytype(enum pkey_key_type key_type) +{ + switch (key_type) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + case PKEY_TYPE_EP11_ECC: + return true; + default: + return false; + } +} + +static int ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (!flags) + flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + if (keylen < sizeof(struct keytoken_header) || flags == 0) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + (hdr->version == TOKVER_EP11_AES_WITH_HEADER || + hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + struct ep11keyblob *kb = (struct ep11keyblob *) + (key + sizeof(struct ep11kblob_header)); + int minhwtype = 0, api = 0; + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + } + rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp, xflags); + if (rc) + goto out; + + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + int minhwtype = 0, api = 0; + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + } + rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp, xflags); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int ep11_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags) +{ + u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + zcrypt_wait_api_operational(); + + if (ktype == PKEY_TYPE_EP11 || + ktype == PKEY_TYPE_EP11_AES || + ktype == PKEY_TYPE_EP11_ECC) { + u8 *wkvp = NULL; + int api; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + wkvp = cur_mkvp; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, api, wkvp, xflags); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, (int)ktype); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int ep11_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + u32 xflags; + int i, rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 AES key blob with header */ + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_ECC_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 ECC key blob with header */ + if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + /* EP11 AES key blob with header in session field */ + if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = ep11_apqns4key(key, keylen, 0, _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype, xflags); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_ECC_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype, xflags); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype, xflags); + } else { + rc = -EINVAL; + break; + } + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate EP11 secure key. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (results in subtype PKEY_TYPE_EP11_AES), + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int ep11_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags) +{ + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + int i, len, rc; + u32 xflags; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = ep11_apqns4type(subtype, NULL, NULL, 0, + _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = ep11_genaeskey(apqns[i].card, apqns[i].domain, + keybitsize, flags, + keybuf, keybuflen, subtype, xflags); + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate EP11 secure key with given clear key value. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (assumes PKEY_TYPE_EP11_AES then). + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int ep11_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags) +{ + struct pkey_apqn _apqns[MAXAPQNSINLIST]; + int i, len, rc; + u32 xflags; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + rc = ep11_apqns4type(subtype, NULL, NULL, 0, + _apqns, &nr_apqns, pflags); + if (rc) + goto out; + apqns = _apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = ep11_clr2keyblob(apqns[i].card, apqns[i].domain, + keybitsize, flags, clrkey, + keybuf, keybuflen, subtype, xflags); + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int ep11_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags, u32 pflags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 apqns[MAXAPQNSINLIST], nr_apqns = ARRAY_SIZE(apqns); + u32 xflags; + int rc; + + xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + int api; + + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11; + *keybitsize = kb->head.bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen), xflags); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER) { + struct ep11kblob_header *kh = (struct ep11kblob_header *)key; + int api; + + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11_AES; + *keybitsize = kh->bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen), xflags); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * This function provides an alternate but usually slow way + * to convert a 'clear key token' with AES key material into + * a protected key. That is done via an intermediate step + * which creates an EP11 AES secure key first and then derives + * the protected key from this secure key. + */ +static int ep11_slowpath_key2protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype, u32 pflags) +{ + const struct keytoken_header *hdr = (const struct keytoken_header *)key; + const struct clearkeytoken *t = (const struct clearkeytoken *)key; + u8 tmpbuf[MAXEP11AESKEYBLOBSIZE]; /* 336 bytes */ + u32 tmplen, keysize = 0; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_CLEAR_KEY) + keysize = pkey_keytype_aes_to_size(t->keytype); + if (!keysize || t->len != keysize) + return -EINVAL; + + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + tmplen = MAXEP11AESKEYBLOBSIZE; + rc = ep11_clr2key(NULL, 0, t->keytype, PKEY_TYPE_EP11, + 8 * keysize, 0, t->clearkey, t->len, + tmpbuf, &tmplen, NULL, pflags); + pr_debug("ep11_clr2key()=%d\n", rc); + if (rc) + continue; + rc = ep11_key2protkey(NULL, 0, tmpbuf, tmplen, + protkey, protkeylen, protkeytype, pflags); + pr_debug("ep11_key2protkey()=%d\n", rc); + } + + pr_debug("rc=%d\n", rc); + return rc; +} + +static struct pkey_handler ep11_handler = { + .module = THIS_MODULE, + .name = "PKEY EP11 handler", + .is_supported_key = is_ep11_key, + .is_supported_keytype = is_ep11_keytype, + .key_to_protkey = ep11_key2protkey, + .slowpath_key_to_protkey = ep11_slowpath_key2protkey, + .gen_key = ep11_gen_key, + .clr_to_key = ep11_clr2key, + .verify_key = ep11_verifykey, + .apqns_for_key = ep11_apqns4key, + .apqns_for_keytype = ep11_apqns4type, +}; + +/* + * Module init + */ +static int __init pkey_ep11_init(void) +{ + /* register this module as pkey handler for all the ep11 stuff */ + return pkey_handler_register(&ep11_handler); +} + +/* + * Module exit + */ +static void __exit pkey_ep11_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&ep11_handler); +} + +module_init(pkey_ep11_init); +module_exit(pkey_ep11_exit); diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c new file mode 100644 index 000000000000..793326c4c59a --- /dev/null +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey pckmo specific code + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/cpufeature.h> +#include <asm/cpacf.h> +#include <crypto/aes.h> +#include <linux/random.h> + +#include "zcrypt_ccamisc.h" +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key PCKMO handler"); + +/* + * Check key blob for known and supported here. + */ +static bool is_pckmo_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct clearkeytoken *t = (struct clearkeytoken *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_NON_CCA: + switch (hdr->version) { + case TOKVER_CLEAR_KEY: + if (pkey_keytype_to_size(t->keytype)) + return true; + return false; + case TOKVER_PROTECTED_KEY: + return true; + default: + return false; + } + default: + return false; + } +} + +static bool is_pckmo_keytype(enum pkey_key_type keytype) +{ + switch (keytype) { + case PKEY_TYPE_PROTKEY: + return true; + default: + return false; + } +} + +/* + * Create a protected key from a clear key value via PCKMO instruction. + */ +static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + /* mask of available pckmo subfunctions */ + static cpacf_mask_t pckmo_functions; + + int keysize, rc = -EINVAL; + u8 paramblock[160]; + u32 pkeytype = 0; + unsigned int fc; + + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + fc = CPACF_PCKMO_ENC_AES_128_KEY; + break; + case PKEY_KEYTYPE_AES_192: + fc = CPACF_PCKMO_ENC_AES_192_KEY; + break; + case PKEY_KEYTYPE_AES_256: + fc = CPACF_PCKMO_ENC_AES_256_KEY; + break; + case PKEY_KEYTYPE_ECC_P256: + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P256_KEY; + break; + case PKEY_KEYTYPE_ECC_P384: + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P384_KEY; + break; + case PKEY_KEYTYPE_ECC_P521: + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P521_KEY; + break; + case PKEY_KEYTYPE_ECC_ED25519: + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY; + break; + case PKEY_KEYTYPE_ECC_ED448: + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; + break; + case PKEY_KEYTYPE_AES_XTS_128: + fc = CPACF_PCKMO_ENC_AES_XTS_128_DOUBLE_KEY; + break; + case PKEY_KEYTYPE_AES_XTS_256: + fc = CPACF_PCKMO_ENC_AES_XTS_256_DOUBLE_KEY; + break; + case PKEY_KEYTYPE_HMAC_512: + fc = CPACF_PCKMO_ENC_HMAC_512_KEY; + break; + case PKEY_KEYTYPE_HMAC_1024: + fc = CPACF_PCKMO_ENC_HMAC_1024_KEY; + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, keytype); + goto out; + } + + keysize = pkey_keytype_to_size(keytype); + pkeytype = pkeytype ?: keytype; + + if (clrkeylen && clrkeylen < keysize) { + PKEY_DBF_ERR("%s clear key size too small: %u < %d\n", + __func__, clrkeylen, keysize); + goto out; + } + if (*protkeylen < keysize + AES_WK_VP_SIZE) { + PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", + __func__, *protkeylen, keysize + AES_WK_VP_SIZE); + goto out; + } + + /* Did we already check for PCKMO ? */ + if (!pckmo_functions.bytes[0]) { + /* no, so check now */ + if (!cpacf_query(CPACF_PCKMO, &pckmo_functions)) { + PKEY_DBF_ERR("%s cpacf_query() failed\n", __func__); + rc = -ENODEV; + goto out; + } + } + /* check for the pckmo subfunction we need now */ + if (!cpacf_test_func(&pckmo_functions, fc)) { + PKEY_DBF_ERR("%s pckmo fc 0x%02x not available\n", + __func__, fc); + rc = -ENODEV; + goto out; + } + + /* prepare param block */ + memset(paramblock, 0, sizeof(paramblock)); + memcpy(paramblock, clrkey, keysize); + + /* call the pckmo instruction */ + cpacf_pckmo(fc, paramblock); + + /* copy created protected key to key buffer including the wkvp block */ + *protkeylen = keysize + AES_WK_VP_SIZE; + memcpy(protkey, paramblock, *protkeylen); + *protkeytype = pkeytype; + + rc = 0; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Verify a raw protected key blob. + */ +static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype) +{ + u8 clrkey[16] = { 0 }, tmpkeybuf[16 + AES_WK_VP_SIZE]; + u32 tmpkeybuflen, tmpkeytype; + int keysize, rc = -EINVAL; + u8 *wkvp; + + /* check protkey type and size */ + keysize = pkey_keytype_to_size(protkeytype); + if (!keysize) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, + protkeytype); + goto out; + } + if (protkeylen < keysize + AES_WK_VP_SIZE) + goto out; + + /* generate a dummy AES 128 protected key */ + tmpkeybuflen = sizeof(tmpkeybuf); + rc = pckmo_clr2protkey(PKEY_KEYTYPE_AES_128, + clrkey, sizeof(clrkey), + tmpkeybuf, &tmpkeybuflen, &tmpkeytype); + if (rc) + goto out; + memzero_explicit(tmpkeybuf, 16); + wkvp = tmpkeybuf + 16; + + /* compare WK VP from the temp key with that of the given prot key */ + if (memcmp(wkvp, protkey + keysize, AES_WK_VP_SIZE)) { + PKEY_DBF_ERR("%s protected key WK VP mismatch\n", __func__); + rc = -EKEYREJECTED; + goto out; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc = -EINVAL; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + if (hdr->type != TOKTYPE_NON_CCA) + return -EINVAL; + + switch (hdr->version) { + case TOKVER_PROTECTED_KEY: { + struct protkeytoken *t = (struct protkeytoken *)key; + u32 keysize; + + if (keylen < sizeof(*t)) + goto out; + keysize = pkey_keytype_to_size(t->keytype); + if (!keysize) { + PKEY_DBF_ERR("%s protected key token: unknown keytype %u\n", + __func__, t->keytype); + goto out; + } + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + if (t->len != keysize + AES_WK_VP_SIZE || + keylen < sizeof(struct protaeskeytoken)) + goto out; + rc = pckmo_verify_protkey(t->protkey, t->len, + t->keytype); + if (rc) + goto out; + break; + default: + if (t->len != keysize + AES_WK_VP_SIZE || + keylen < sizeof(*t) + keysize + AES_WK_VP_SIZE) + goto out; + break; + } + memcpy(protkey, t->protkey, t->len); + *protkeylen = t->len; + *protkeytype = t->keytype; + rc = 0; + break; + } + case TOKVER_CLEAR_KEY: { + struct clearkeytoken *t = (struct clearkeytoken *)key; + u32 keysize; + + if (keylen < sizeof(*t) || + keylen < sizeof(*t) + t->len) + goto out; + keysize = pkey_keytype_to_size(t->keytype); + if (!keysize) { + PKEY_DBF_ERR("%s clear key token: unknown keytype %u\n", + __func__, t->keytype); + goto out; + } + if (t->len != keysize) { + PKEY_DBF_ERR("%s clear key token: invalid key len %u\n", + __func__, t->len); + goto out; + } + rc = pckmo_clr2protkey(t->keytype, t->clearkey, t->len, + protkey, protkeylen, protkeytype); + break; + } + default: + PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", + __func__, hdr->version); + break; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate a random protected key. + */ +static int pckmo_gen_protkey(u32 keytype, u32 subtype, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + u8 clrkey[128]; + int keysize; + int rc; + + keysize = pkey_keytype_to_size(keytype); + if (!keysize) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + if (subtype != PKEY_TYPE_PROTKEY) { + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + case PKEY_KEYTYPE_AES_XTS_128: + case PKEY_KEYTYPE_AES_XTS_256: + case PKEY_KEYTYPE_HMAC_512: + case PKEY_KEYTYPE_HMAC_1024: + break; + default: + PKEY_DBF_ERR("%s unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + /* generate a dummy random clear key */ + get_random_bytes(clrkey, keysize); + + /* convert it to a dummy protected key */ + rc = pckmo_clr2protkey(keytype, clrkey, keysize, + protkey, protkeylen, protkeytype); + if (rc) + goto out; + + /* replace the key part of the protected key with random bytes */ + get_random_bytes(protkey, keysize); + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Verify a protected key token blob. + */ +static int pckmo_verify_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc = -EINVAL; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + if (hdr->type != TOKTYPE_NON_CCA) + return -EINVAL; + + switch (hdr->version) { + case TOKVER_PROTECTED_KEY: { + struct protkeytoken *t = (struct protkeytoken *)key; + u32 keysize; + + if (keylen < sizeof(*t)) + goto out; + keysize = pkey_keytype_to_size(t->keytype); + if (!keysize || t->len != keysize + AES_WK_VP_SIZE) + goto out; + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + if (keylen < sizeof(struct protaeskeytoken)) + goto out; + break; + default: + if (keylen < sizeof(*t) + keysize + AES_WK_VP_SIZE) + goto out; + break; + } + rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); + break; + } + default: + PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", + __func__, hdr->version); + break; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Wrapper functions used for the pkey handler struct + */ + +static int pkey_pckmo_key2protkey(const struct pkey_apqn *_apqns, + size_t _nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *keyinfo, + u32 _xflags __always_unused) +{ + return pckmo_key2protkey(key, keylen, + protkey, protkeylen, keyinfo); +} + +static int pkey_pckmo_gen_key(const struct pkey_apqn *_apqns, size_t _nr_apqns, + u32 keytype, u32 keysubtype, + u32 _keybitsize, u32 _flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo, + u32 _xflags __always_unused) +{ + return pckmo_gen_protkey(keytype, keysubtype, + keybuf, keybuflen, keyinfo); +} + +static int pkey_pckmo_verifykey(const u8 *key, u32 keylen, + u16 *_card, u16 *_dom, + u32 *_keytype, u32 *_keybitsize, + u32 *_flags, u32 _xflags __always_unused) +{ + return pckmo_verify_key(key, keylen); +} + +static struct pkey_handler pckmo_handler = { + .module = THIS_MODULE, + .name = "PKEY PCKMO handler", + .is_supported_key = is_pckmo_key, + .is_supported_keytype = is_pckmo_keytype, + .key_to_protkey = pkey_pckmo_key2protkey, + .gen_key = pkey_pckmo_gen_key, + .verify_key = pkey_pckmo_verifykey, +}; + +/* + * Module init + */ +static int __init pkey_pckmo_init(void) +{ + cpacf_mask_t func_mask; + + /* + * The pckmo instruction should be available - even if we don't + * actually invoke it. This instruction comes with MSA 3 which + * is also the minimum level for the kmc instructions which + * are able to work with protected keys. + */ + if (!cpacf_query(CPACF_PCKMO, &func_mask)) + return -ENODEV; + + /* register this module as pkey handler for all the pckmo stuff */ + return pkey_handler_register(&pckmo_handler); +} + +/* + * Module exit + */ +static void __exit pkey_pckmo_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&pckmo_handler); +} + +module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_pckmo_init); +module_exit(pkey_pckmo_exit); diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c new file mode 100644 index 000000000000..b6b0a46cb8a8 --- /dev/null +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -0,0 +1,646 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey module sysfs related functions + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/sysfs.h> + +#include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" + +#include "pkey_base.h" + +/* + * Wrapper around pkey_handler_gen_key() which deals with the + * ENODEV return code and then tries to enforce a pkey handler + * module load. + */ +static int sys_pkey_handler_gen_key(u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + int rc; + + rc = pkey_handler_gen_key(NULL, 0, + keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo, 0); + if (rc == -ENODEV) { + pkey_handler_request_modules(); + rc = pkey_handler_gen_key(NULL, 0, + keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo, 0); + } + + return rc; +} + +/* + * Sysfs attribute read function for all protected key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, + loff_t off, size_t count) +{ + struct protaeskeytoken protkeytoken; + struct pkey_protkey protkey; + int rc; + + if (off != 0 || count < sizeof(protkeytoken)) + return -EINVAL; + if (is_xts) + if (count < 2 * sizeof(protkeytoken)) + return -EINVAL; + + memset(&protkeytoken, 0, sizeof(protkeytoken)); + protkeytoken.type = TOKTYPE_NON_CCA; + protkeytoken.version = TOKVER_PROTECTED_KEY; + protkeytoken.keytype = keytype; + + protkey.len = sizeof(protkey.protkey); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); + if (rc) + return rc; + + protkeytoken.len = protkey.len; + memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); + + memcpy(buf, &protkeytoken, sizeof(protkeytoken)); + + if (is_xts) { + /* xts needs a second protected key, reuse protkey struct */ + protkey.len = sizeof(protkey.protkey); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); + if (rc) + return rc; + + protkeytoken.len = protkey.len; + memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); + + memcpy(buf + sizeof(protkeytoken), &protkeytoken, + sizeof(protkeytoken)); + + return 2 * sizeof(protkeytoken); + } + + return sizeof(protkeytoken); +} + +/* + * Sysfs attribute read function for the AES XTS prot key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_aes_xts_attr_read(u32 keytype, char *buf, + loff_t off, size_t count) +{ + struct protkeytoken *t = (struct protkeytoken *)buf; + u32 protlen, prottype; + int rc; + + switch (keytype) { + case PKEY_KEYTYPE_AES_XTS_128: + protlen = 64; + break; + case PKEY_KEYTYPE_AES_XTS_256: + protlen = 96; + break; + default: + return -EINVAL; + } + + if (off != 0 || count < sizeof(*t) + protlen) + return -EINVAL; + + memset(t, 0, sizeof(*t) + protlen); + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + t->protkey, &protlen, &prottype); + if (rc) + return rc; + + t->len = protlen; + + return sizeof(*t) + protlen; +} + +/* + * Sysfs attribute read function for the HMAC prot key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_hmac_attr_read(u32 keytype, char *buf, + loff_t off, size_t count) +{ + struct protkeytoken *t = (struct protkeytoken *)buf; + u32 protlen, prottype; + int rc; + + switch (keytype) { + case PKEY_KEYTYPE_HMAC_512: + protlen = 96; + break; + case PKEY_KEYTYPE_HMAC_1024: + protlen = 160; + break; + default: + return -EINVAL; + } + + if (off != 0 || count < sizeof(*t) + protlen) + return -EINVAL; + + memset(t, 0, sizeof(*t) + protlen); + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + t->protkey, &protlen, &prottype); + if (rc) + return rc; + + t->len = protlen; + + return sizeof(*t) + protlen; +} + +static ssize_t protkey_aes_128_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, + off, count); +} + +static ssize_t protkey_aes_192_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, + off, count); +} + +static ssize_t protkey_aes_256_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, + off, count); +} + +static ssize_t protkey_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, + off, count); +} + +static ssize_t protkey_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, + off, count); +} + +static ssize_t protkey_aes_xts_128_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_128, + buf, off, count); +} + +static ssize_t protkey_aes_xts_256_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_256, + buf, off, count); +} + +static ssize_t protkey_hmac_512_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_512, + buf, off, count); +} + +static ssize_t protkey_hmac_1024_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_1024, + buf, off, count); +} + +static const BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken)); +static const BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken)); +static const BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken)); +static const BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken)); +static const BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken)); +static const BIN_ATTR_RO(protkey_aes_xts_128, sizeof(struct protkeytoken) + 64); +static const BIN_ATTR_RO(protkey_aes_xts_256, sizeof(struct protkeytoken) + 96); +static const BIN_ATTR_RO(protkey_hmac_512, sizeof(struct protkeytoken) + 96); +static const BIN_ATTR_RO(protkey_hmac_1024, sizeof(struct protkeytoken) + 160); + +static const struct bin_attribute *const protkey_attrs[] = { + &bin_attr_protkey_aes_128, + &bin_attr_protkey_aes_192, + &bin_attr_protkey_aes_256, + &bin_attr_protkey_aes_128_xts, + &bin_attr_protkey_aes_256_xts, + &bin_attr_protkey_aes_xts_128, + &bin_attr_protkey_aes_xts_256, + &bin_attr_protkey_hmac_512, + &bin_attr_protkey_hmac_1024, + NULL +}; + +static const struct attribute_group protkey_attr_group = { + .name = "protkey", + .bin_attrs = protkey_attrs, +}; + +/* + * Sysfs attribute read function for all secure key ccadata binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, + loff_t off, size_t count) +{ + struct pkey_seckey *seckey = (struct pkey_seckey *)buf; + u32 buflen; + int rc; + + if (off != 0 || count < sizeof(struct secaeskeytoken)) + return -EINVAL; + if (is_xts) + if (count < 2 * sizeof(struct secaeskeytoken)) + return -EINVAL; + + buflen = sizeof(seckey->seckey); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); + if (rc) + return rc; + + if (is_xts) { + seckey++; + buflen = sizeof(seckey->seckey); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); + if (rc) + return rc; + + return 2 * sizeof(struct secaeskeytoken); + } + + return sizeof(struct secaeskeytoken); +} + +static ssize_t ccadata_aes_128_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, + off, count); +} + +static ssize_t ccadata_aes_192_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, + off, count); +} + +static ssize_t ccadata_aes_256_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, + off, count); +} + +static ssize_t ccadata_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, + off, count); +} + +static ssize_t ccadata_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, + off, count); +} + +static const BIN_ATTR_RO(ccadata_aes_128, sizeof(struct secaeskeytoken)); +static const BIN_ATTR_RO(ccadata_aes_192, sizeof(struct secaeskeytoken)); +static const BIN_ATTR_RO(ccadata_aes_256, sizeof(struct secaeskeytoken)); +static const BIN_ATTR_RO(ccadata_aes_128_xts, 2 * sizeof(struct secaeskeytoken)); +static const BIN_ATTR_RO(ccadata_aes_256_xts, 2 * sizeof(struct secaeskeytoken)); + +static const struct bin_attribute *const ccadata_attrs[] = { + &bin_attr_ccadata_aes_128, + &bin_attr_ccadata_aes_192, + &bin_attr_ccadata_aes_256, + &bin_attr_ccadata_aes_128_xts, + &bin_attr_ccadata_aes_256_xts, + NULL +}; + +static const struct attribute_group ccadata_attr_group = { + .name = "ccadata", + .bin_attrs = ccadata_attrs, +}; + +#define CCACIPHERTOKENSIZE (sizeof(struct cipherkeytoken) + 80) + +/* + * Sysfs attribute read function for all secure key ccacipher binary attributes. + * The implementation can not deal with partial reads, because a new random + * secure key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, + bool is_xts, char *buf, loff_t off, + size_t count) +{ + u32 keysize = CCACIPHERTOKENSIZE; + int rc; + + if (off != 0 || count < CCACIPHERTOKENSIZE) + return -EINVAL; + if (is_xts) + if (count < 2 * CCACIPHERTOKENSIZE) + return -EINVAL; + + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); + if (rc) + return rc; + + if (is_xts) { + keysize = CCACIPHERTOKENSIZE; + buf += CCACIPHERTOKENSIZE; + rc = sys_pkey_handler_gen_key( + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); + if (rc) + return rc; + return 2 * CCACIPHERTOKENSIZE; + } + + return CCACIPHERTOKENSIZE; +} + +static ssize_t ccacipher_aes_128_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_192_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_192, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_256_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, true, buf, + off, count); +} + +static ssize_t ccacipher_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, true, buf, + off, count); +} + +static const BIN_ATTR_RO(ccacipher_aes_128, CCACIPHERTOKENSIZE); +static const BIN_ATTR_RO(ccacipher_aes_192, CCACIPHERTOKENSIZE); +static const BIN_ATTR_RO(ccacipher_aes_256, CCACIPHERTOKENSIZE); +static const BIN_ATTR_RO(ccacipher_aes_128_xts, 2 * CCACIPHERTOKENSIZE); +static const BIN_ATTR_RO(ccacipher_aes_256_xts, 2 * CCACIPHERTOKENSIZE); + +static const struct bin_attribute *const ccacipher_attrs[] = { + &bin_attr_ccacipher_aes_128, + &bin_attr_ccacipher_aes_192, + &bin_attr_ccacipher_aes_256, + &bin_attr_ccacipher_aes_128_xts, + &bin_attr_ccacipher_aes_256_xts, + NULL +}; + +static const struct attribute_group ccacipher_attr_group = { + .name = "ccacipher", + .bin_attrs = ccacipher_attrs, +}; + +/* + * Sysfs attribute read function for all ep11 aes key binary attributes. + * The implementation can not deal with partial reads, because a new random + * secure key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + * This function and the sysfs attributes using it provide EP11 key blobs + * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently + * 336 bytes. + */ +static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, + bool is_xts, char *buf, loff_t off, + size_t count) +{ + u32 keysize = MAXEP11AESKEYBLOBSIZE; + int rc; + + if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + if (is_xts) + if (count < 2 * MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); + if (rc) + return rc; + + if (is_xts) { + keysize = MAXEP11AESKEYBLOBSIZE; + buf += MAXEP11AESKEYBLOBSIZE; + rc = sys_pkey_handler_gen_key( + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); + if (rc) + return rc; + return 2 * MAXEP11AESKEYBLOBSIZE; + } + + return MAXEP11AESKEYBLOBSIZE; +} + +static ssize_t ep11_aes_128_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf, + off, count); +} + +static ssize_t ep11_aes_192_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf, + off, count); +} + +static ssize_t ep11_aes_256_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf, + off, count); +} + +static ssize_t ep11_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf, + off, count); +} + +static ssize_t ep11_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf, + off, count); +} + +static const BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE); +static const BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE); +static const BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE); +static const BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE); +static const BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE); + +static const struct bin_attribute *const ep11_attrs[] = { + &bin_attr_ep11_aes_128, + &bin_attr_ep11_aes_192, + &bin_attr_ep11_aes_256, + &bin_attr_ep11_aes_128_xts, + &bin_attr_ep11_aes_256_xts, + NULL +}; + +static const struct attribute_group ep11_attr_group = { + .name = "ep11", + .bin_attrs = ep11_attrs, +}; + +const struct attribute_group *pkey_attr_groups[] = { + &protkey_attr_group, + &ccadata_attr_group, + &ccacipher_attr_group, + &ep11_attr_group, + NULL, +}; diff --git a/drivers/s390/crypto/pkey_uv.c b/drivers/s390/crypto/pkey_uv.c new file mode 100644 index 000000000000..6cd3c49384b5 --- /dev/null +++ b/drivers/s390/crypto/pkey_uv.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey uv specific code + * + * Copyright IBM Corp. 2024 + */ + +#define pr_fmt(fmt) "pkey: " fmt + +#include <linux/cpufeature.h> +#include <linux/init.h> +#include <linux/module.h> +#include <asm/uv.h> + +#include "zcrypt_ccamisc.h" +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key UV handler"); + +/* + * One pre-allocated uv_secret_list for use with uv_find_secret() + */ +static struct uv_secret_list *uv_list; +static DEFINE_MUTEX(uv_list_mutex); + +/* + * UV secret token struct and defines. + */ + +#define TOKVER_UV_SECRET 0x09 + +struct uvsecrettoken { + u8 type; /* 0x00 = TOKTYPE_NON_CCA */ + u8 res0[3]; + u8 version; /* 0x09 = TOKVER_UV_SECRET */ + u8 res1[3]; + u16 secret_type; /* one of enum uv_secret_types from uv.h */ + u16 secret_len; /* length in bytes of the secret */ + u8 secret_id[UV_SECRET_ID_LEN]; /* the secret id for this secret */ +} __packed; + +/* + * Check key blob for known and supported UV key. + */ +static bool is_uv_key(const u8 *key, u32 keylen) +{ + struct uvsecrettoken *t = (struct uvsecrettoken *)key; + + if (keylen < sizeof(*t)) + return false; + + switch (t->type) { + case TOKTYPE_NON_CCA: + switch (t->version) { + case TOKVER_UV_SECRET: + switch (t->secret_type) { + case UV_SECRET_AES_128: + case UV_SECRET_AES_192: + case UV_SECRET_AES_256: + case UV_SECRET_AES_XTS_128: + case UV_SECRET_AES_XTS_256: + case UV_SECRET_HMAC_SHA_256: + case UV_SECRET_HMAC_SHA_512: + case UV_SECRET_ECDSA_P256: + case UV_SECRET_ECDSA_P384: + case UV_SECRET_ECDSA_P521: + case UV_SECRET_ECDSA_ED25519: + case UV_SECRET_ECDSA_ED448: + return true; + default: + return false; + } + default: + return false; + } + default: + return false; + } +} + +static bool is_uv_keytype(enum pkey_key_type keytype) +{ + switch (keytype) { + case PKEY_TYPE_UVSECRET: + return true; + default: + return false; + } +} + +static int get_secret_metadata(const u8 secret_id[UV_SECRET_ID_LEN], + struct uv_secret_list_item_hdr *secret) +{ + int rc; + + mutex_lock(&uv_list_mutex); + memset(uv_list, 0, sizeof(*uv_list)); + rc = uv_find_secret(secret_id, uv_list, secret); + mutex_unlock(&uv_list_mutex); + + return rc; +} + +static int retrieve_secret(const u8 secret_id[UV_SECRET_ID_LEN], + u16 *secret_type, u8 *buf, u32 *buflen) +{ + struct uv_secret_list_item_hdr secret_meta_data; + int rc; + + rc = get_secret_metadata(secret_id, &secret_meta_data); + if (rc) + return rc; + + if (*buflen < secret_meta_data.length) + return -EINVAL; + + rc = uv_retrieve_secret(secret_meta_data.index, + buf, secret_meta_data.length); + if (rc) + return rc; + + *secret_type = secret_meta_data.type; + *buflen = secret_meta_data.length; + + return 0; +} + +static int uv_get_size_and_type(u16 secret_type, u32 *pkeysize, u32 *pkeytype) +{ + int rc = 0; + + switch (secret_type) { + case UV_SECRET_AES_128: + *pkeysize = 16 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_AES_128; + break; + case UV_SECRET_AES_192: + *pkeysize = 24 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_AES_192; + break; + case UV_SECRET_AES_256: + *pkeysize = 32 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_AES_256; + break; + case UV_SECRET_AES_XTS_128: + *pkeysize = 16 + 16 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_AES_XTS_128; + break; + case UV_SECRET_AES_XTS_256: + *pkeysize = 32 + 32 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_AES_XTS_256; + break; + case UV_SECRET_HMAC_SHA_256: + *pkeysize = 64 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_HMAC_512; + break; + case UV_SECRET_HMAC_SHA_512: + *pkeysize = 128 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_HMAC_1024; + break; + case UV_SECRET_ECDSA_P256: + *pkeysize = 32 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_ECC_P256; + break; + case UV_SECRET_ECDSA_P384: + *pkeysize = 48 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_ECC_P384; + break; + case UV_SECRET_ECDSA_P521: + *pkeysize = 80 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_ECC_P521; + break; + case UV_SECRET_ECDSA_ED25519: + *pkeysize = 32 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_ECC_ED25519; + break; + case UV_SECRET_ECDSA_ED448: + *pkeysize = 64 + AES_WK_VP_SIZE; + *pkeytype = PKEY_KEYTYPE_ECC_ED448; + break; + default: + rc = -EINVAL; + } + + return rc; +} + +static int uv_key2protkey(const struct pkey_apqn *_apqns __always_unused, + size_t _nr_apqns __always_unused, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *keyinfo, + u32 _xflags __always_unused) +{ + struct uvsecrettoken *t = (struct uvsecrettoken *)key; + u32 pkeysize, pkeytype; + u16 secret_type; + int rc; + + rc = uv_get_size_and_type(t->secret_type, &pkeysize, &pkeytype); + if (rc) + goto out; + + if (*protkeylen < pkeysize) { + PKEY_DBF_ERR("%s prot key buffer size too small: %u < %u\n", + __func__, *protkeylen, pkeysize); + rc = -EINVAL; + goto out; + } + + rc = retrieve_secret(t->secret_id, &secret_type, protkey, protkeylen); + if (rc) { + PKEY_DBF_ERR("%s retrieve_secret() failed with %d\n", + __func__, rc); + goto out; + } + if (secret_type != t->secret_type) { + PKEY_DBF_ERR("%s retrieved secret type %u != expected type %u\n", + __func__, secret_type, t->secret_type); + rc = -EINVAL; + goto out; + } + + if (keyinfo) + *keyinfo = pkeytype; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static int uv_verifykey(const u8 *key, u32 keylen, + u16 *_card __always_unused, + u16 *_dom __always_unused, + u32 *keytype, u32 *keybitsize, u32 *flags, + u32 xflags __always_unused) +{ + struct uvsecrettoken *t = (struct uvsecrettoken *)key; + struct uv_secret_list_item_hdr secret_meta_data; + u32 pkeysize, pkeytype, bitsize; + int rc; + + rc = uv_get_size_and_type(t->secret_type, &pkeysize, &pkeytype); + if (rc) + goto out; + + rc = get_secret_metadata(t->secret_id, &secret_meta_data); + if (rc) + goto out; + + if (secret_meta_data.type != t->secret_type) { + rc = -EINVAL; + goto out; + } + + /* set keytype; keybitsize and flags are not supported */ + if (keytype) + *keytype = PKEY_TYPE_UVSECRET; + if (keybitsize) { + bitsize = 8 * pkey_keytype_to_size(pkeytype); + *keybitsize = bitsize ?: PKEY_SIZE_UNKNOWN; + } + if (flags) + *flags = pkeytype; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +static struct pkey_handler uv_handler = { + .module = THIS_MODULE, + .name = "PKEY UV handler", + .is_supported_key = is_uv_key, + .is_supported_keytype = is_uv_keytype, + .key_to_protkey = uv_key2protkey, + .verify_key = uv_verifykey, +}; + +/* + * Module init + */ +static int __init pkey_uv_init(void) +{ + int rc; + + if (!is_prot_virt_guest()) + return -ENODEV; + + if (!test_bit_inv(BIT_UVC_CMD_RETR_SECRET, uv_info.inst_calls_list)) + return -ENODEV; + + uv_list = kmalloc(sizeof(*uv_list), GFP_KERNEL); + if (!uv_list) + return -ENOMEM; + + rc = pkey_handler_register(&uv_handler); + if (rc) + kfree(uv_list); + + return rc; +} + +/* + * Module exit + */ +static void __exit pkey_uv_exit(void) +{ + pkey_handler_unregister(&uv_handler); + mutex_lock(&uv_list_mutex); + kvfree(uv_list); + mutex_unlock(&uv_list_mutex); +} + +module_cpu_feature_match(S390_CPU_FEATURE_UV, pkey_uv_init); +module_exit(pkey_uv_exit); diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index a5ab03e42ff1..67a807e2e75b 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -26,6 +26,18 @@ MODULE_LICENSE("GPL v2"); struct ap_matrix_dev *matrix_dev; debug_info_t *vfio_ap_dbf_info; +static ssize_t features_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "guest_matrix hotplug ap_config\n"); +} +static DEVICE_ATTR_RO(features); + +static struct attribute *matrix_dev_attrs[] = { + &dev_attr_features.attr, + NULL, +}; +ATTRIBUTE_GROUPS(matrix_dev); + /* Only type 10 adapters (CEX4 and later) are supported * by the AP matrix device driver */ @@ -60,7 +72,7 @@ static void vfio_ap_matrix_dev_release(struct device *dev) kfree(matrix_dev); } -static struct bus_type matrix_bus = { +static const struct bus_type matrix_bus = { .name = "matrix", }; @@ -68,6 +80,7 @@ static struct device_driver matrix_driver = { .name = "vfio_ap", .bus = &matrix_bus, .suppress_bind_attrs = true, + .dev_groups = matrix_dev_groups, }; static int vfio_ap_matrix_dev_create(void) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 0509f80622cd..48da32ad0493 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -32,7 +32,8 @@ #define AP_RESET_INTERVAL 20 /* Reset sleep interval (20ms) */ -static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable); +static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); +static int vfio_ap_mdev_reset_qlist(struct list_head *qlist); static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); @@ -353,16 +354,32 @@ static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib) if (!*nib) return -EINVAL; - if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT))) + if (!kvm_s390_is_gpa_in_memslot(vcpu->kvm, *nib)) return -EINVAL; return 0; } -static int ensure_nib_shared(unsigned long addr, struct gmap *gmap) +/** + * ensure_nib_shared() - Ensure the address of the NIB is secure and shared + * @addr: the physical (absolute) address of the NIB + * + * This function checks whether the NIB page, which has been pinned with + * vfio_pin_pages(), is a shared page belonging to a secure guest. + * + * It will call uv_pin_shared() on it; if the page was already pinned shared + * (i.e. if the NIB belongs to a secure guest and is shared), then 0 + * (success) is returned. If the NIB was not shared, vfio_pin_pages() had + * exported it and now it does not belong to the secure guest anymore. In + * that case, an error is returned. + * + * Context: the NIB (at physical address @addr) has to be pinned with + * vfio_pin_pages() before calling this function. + * + * Return: 0 in case of success, otherwise an error < 0. + */ +static int ensure_nib_shared(unsigned long addr) { - int ret; - /* * The nib has to be located in shared storage since guest and * host access it. vfio_pin_pages() will do a pin shared and @@ -373,12 +390,7 @@ static int ensure_nib_shared(unsigned long addr, struct gmap *gmap) * * If the page is already pinned shared the UV will return a success. */ - ret = uv_pin_shared(addr); - if (ret) { - /* vfio_pin_pages() likely exported the page so let's re-import */ - gmap_convert_to_secure(gmap, addr); - } - return ret; + return uv_pin_shared(addr); } /** @@ -393,8 +405,8 @@ static int ensure_nib_shared(unsigned long addr, struct gmap *gmap) * Register the guest ISC to GIB interface and retrieve the * host ISC to issue the host side PQAP/AQIC * - * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the - * vfio_pin_pages failed. + * status.response_code may be set to AP_RESPONSE_INVALID_ADDRESS in case the + * vfio_pin_pages or kvm_s390_gisc_register failed. * * Otherwise return the ap_queue_status returned by the ap_aqic(), * all retry handling will be done by the guest. @@ -424,6 +436,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, return status; } + /* The pin will probably be successful even if the NIB was not shared */ ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1, IOMMU_READ | IOMMU_WRITE, &h_page); switch (ret) { @@ -446,7 +459,7 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, /* NIB in non-shared storage is a rc 6 for PV guests */ if (kvm_s390_pv_cpu_is_protected(vcpu) && - ensure_nib_shared(h_nib & PAGE_MASK, kvm->arch.gmap)) { + ensure_nib_shared(h_nib & PAGE_MASK)) { vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); status.response_code = AP_RESPONSE_INVALID_ADDRESS; return status; @@ -457,7 +470,8 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n", __func__, nisc, isc, q->apqn); - status.response_code = AP_RESPONSE_INVALID_GISA; + vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); + status.response_code = AP_RESPONSE_INVALID_ADDRESS; return status; } @@ -475,8 +489,11 @@ static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q, break; case AP_RESPONSE_OTHERWISE_CHANGED: /* We could not modify IRQ settings: clear new configuration */ + ret = kvm_s390_gisc_unregister(kvm, isc); + if (ret) + VFIO_AP_DBF_WARN("%s: kvm_s390_gisc_unregister: rc=%d isc=%d, apqn=%#04x\n", + __func__, ret, isc, q->apqn); vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1); - kvm_s390_gisc_unregister(kvm, isc); break; default: pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn, @@ -633,13 +650,22 @@ static void vfio_ap_matrix_init(struct ap_config_info *info, matrix->adm_max = info->apxa ? info->nd : 15; } +static void signal_guest_ap_cfg_changed(struct ap_matrix_mdev *matrix_mdev) +{ + if (matrix_mdev->cfg_chg_trigger) + eventfd_signal(matrix_mdev->cfg_chg_trigger); +} + static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev) { - if (matrix_mdev->kvm) + if (matrix_mdev->kvm) { kvm_arch_crypto_set_masks(matrix_mdev->kvm, matrix_mdev->shadow_apcb.apm, matrix_mdev->shadow_apcb.aqm, matrix_mdev->shadow_apcb.adm); + + signal_guest_ap_cfg_changed(matrix_mdev); + } } static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) @@ -654,6 +680,21 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) AP_DOMAINS); } +static bool _queue_passable(struct vfio_ap_queue *q) +{ + if (!q) + return false; + + switch (q->reset_status.response_code) { + case AP_RESPONSE_NORMAL: + case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: + return true; + default: + return false; + } +} + /* * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev * to ensure no queue devices are passed through to @@ -661,26 +702,32 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) * device driver. * * @matrix_mdev: the matrix mdev whose matrix is to be filtered. + * @apm_filtered: a 256-bit bitmap for storing the APIDs filtered from the + * guest's AP configuration that are still in the host's AP + * configuration. * * Note: If an APQN referencing a queue device that is not bound to the vfio_ap * driver, its APID will be filtered from the guest's APCB. The matrix * structure precludes filtering an individual APQN, so its APID will be - * filtered. + * filtered. Consequently, all queues associated with the adapter that + * are in the host's AP configuration must be reset. If queues are + * subsequently made available again to the guest, they should re-appear + * in a reset state * * Return: a boolean value indicating whether the KVM guest's APCB was changed * by the filtering or not. */ -static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, - struct ap_matrix_mdev *matrix_mdev) +static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apm_filtered) { unsigned long apid, apqi, apqn; DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); - struct vfio_ap_queue *q; bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); + bitmap_clear(apm_filtered, 0, AP_DEVICES); /* * Copy the adapters, domains and control domains to the shadow_apcb @@ -692,8 +739,9 @@ static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm, (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS); - for_each_set_bit_inv(apid, apm, AP_DEVICES) { - for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { + for_each_set_bit_inv(apid, matrix_mdev->shadow_apcb.apm, AP_DEVICES) { + for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, + AP_DOMAINS) { /* * If the APQN is not bound to the vfio_ap device * driver, then we can't assign it to the guest's @@ -703,10 +751,17 @@ static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, * hardware device. */ apqn = AP_MKQID(apid, apqi); - q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); - if (!q || q->reset_status.response_code) { - clear_bit_inv(apid, - matrix_mdev->shadow_apcb.apm); + if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) { + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + + /* + * If the adapter was previously plugged into + * the guest, let's let the caller know that + * the APID was filtered. + */ + if (test_bit_inv(apid, prev_shadow_apm)) + set_bit_inv(apid, apm_filtered); + break; } } @@ -746,6 +801,7 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) if (ret) goto err_put_vdev; matrix_mdev->req_trigger = NULL; + matrix_mdev->cfg_chg_trigger = NULL; dev_set_drvdata(&mdev->dev, matrix_mdev); mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); @@ -760,10 +816,11 @@ err_put_vdev: static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, struct vfio_ap_queue *q) { - if (q) { - q->matrix_mdev = matrix_mdev; - hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); - } + if (!q || vfio_ap_mdev_get_queue(matrix_mdev, q->apqn)) + return; + + q->matrix_mdev = matrix_mdev; + hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); } static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) @@ -808,7 +865,7 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); - vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); + vfio_ap_mdev_reset_queues(matrix_mdev); vfio_ap_mdev_unlink_fr_queues(matrix_mdev); list_del(&matrix_mdev->node); mutex_unlock(&matrix_dev->mdevs_lock); @@ -816,48 +873,66 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) vfio_put_device(&matrix_mdev->vdev); } -#define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ - "already assigned to %s" +#define MDEV_SHARING_ERR "Userspace may not assign queue %02lx.%04lx to mdev: already assigned to %s" + +#define MDEV_IN_USE_ERR "Can not reserve queue %02lx.%04lx for host driver: in use by mdev" + +static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *assignee, + struct ap_matrix_mdev *assigned_to, + unsigned long *apm, unsigned long *aqm) +{ + unsigned long apid, apqi; + + for_each_set_bit_inv(apid, apm, AP_DEVICES) { + for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { + dev_warn(mdev_dev(assignee->mdev), MDEV_SHARING_ERR, + apid, apqi, dev_name(mdev_dev(assigned_to->mdev))); + } + } +} -static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, - unsigned long *apm, - unsigned long *aqm) +static void vfio_ap_mdev_log_in_use_err(struct ap_matrix_mdev *assignee, + unsigned long *apm, unsigned long *aqm) { unsigned long apid, apqi; - const struct device *dev = mdev_dev(matrix_mdev->mdev); - const char *mdev_name = dev_name(dev); - for_each_set_bit_inv(apid, apm, AP_DEVICES) + for_each_set_bit_inv(apid, apm, AP_DEVICES) { for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) - dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); + dev_warn(mdev_dev(assignee->mdev), MDEV_IN_USE_ERR, apid, apqi); + } } /** * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs * + * @assignee: the matrix mdev to which @mdev_apm and @mdev_aqm are being + * assigned; or, NULL if this function was called by the AP bus + * driver in_use callback to verify none of the APQNs being reserved + * for the host device driver are in use by a vfio_ap mediated device * @mdev_apm: mask indicating the APIDs of the APQNs to be verified * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified * - * Verifies that each APQN derived from the Cartesian product of a bitmap of - * AP adapter IDs and AP queue indexes is not configured for any matrix - * mediated device. AP queue sharing is not allowed. + * Verifies that each APQN derived from the Cartesian product of APIDs + * represented by the bits set in @mdev_apm and the APQIs of the bits set in + * @mdev_aqm is not assigned to a mediated device other than the mdev to which + * the APQN is being assigned (@assignee). AP queue sharing is not allowed. * * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. */ -static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, +static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *assignee, + unsigned long *mdev_apm, unsigned long *mdev_aqm) { - struct ap_matrix_mdev *matrix_mdev; + struct ap_matrix_mdev *assigned_to; DECLARE_BITMAP(apm, AP_DEVICES); DECLARE_BITMAP(aqm, AP_DOMAINS); - list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + list_for_each_entry(assigned_to, &matrix_dev->mdev_list, node) { /* - * If the input apm and aqm are fields of the matrix_mdev - * object, then move on to the next matrix_mdev. + * If the mdev to which the mdev_apm and mdev_aqm is being + * assigned is the same as the mdev being verified */ - if (mdev_apm == matrix_mdev->matrix.apm && - mdev_aqm == matrix_mdev->matrix.aqm) + if (assignee == assigned_to) continue; memset(apm, 0, sizeof(apm)); @@ -867,15 +942,16 @@ static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, * We work on full longs, as we can only exclude the leftover * bits in non-inverse order. The leftover is all zeros. */ - if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, - AP_DEVICES)) + if (!bitmap_and(apm, mdev_apm, assigned_to->matrix.apm, AP_DEVICES)) continue; - if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, - AP_DOMAINS)) + if (!bitmap_and(aqm, mdev_aqm, assigned_to->matrix.aqm, AP_DOMAINS)) continue; - vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); + if (assignee) + vfio_ap_mdev_log_sharing_err(assignee, assigned_to, apm, aqm); + else + vfio_ap_mdev_log_in_use_err(assigned_to, apm, aqm); return -EADDRINUSE; } @@ -892,7 +968,7 @@ static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, * * Return: One of the following values: * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, - * most likely -EBUSY indicating the ap_perms_mutex lock is already held. + * most likely -EBUSY indicating the ap_attr_mutex lock is already held. * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the * zcrypt default driver. * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev @@ -904,7 +980,8 @@ static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->matrix.aqm)) return -EADDRNOTAVAIL; - return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, + return vfio_ap_mdev_verify_no_sharing(matrix_mdev, + matrix_mdev->matrix.apm, matrix_mdev->matrix.aqm); } @@ -918,6 +995,47 @@ static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, AP_MKQID(apid, apqi)); } +static void collect_queues_to_reset(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid, + struct list_head *qlist) +{ + struct vfio_ap_queue *q; + unsigned long apqi; + + for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS) { + q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + if (q) + list_add_tail(&q->reset_qnode, qlist); + } +} + +static void reset_queues_for_apid(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + struct list_head qlist; + + INIT_LIST_HEAD(&qlist); + collect_queues_to_reset(matrix_mdev, apid, &qlist); + vfio_ap_mdev_reset_qlist(&qlist); +} + +static int reset_queues_for_apids(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apm_reset) +{ + struct list_head qlist; + unsigned long apid; + + if (bitmap_empty(apm_reset, AP_DEVICES)) + return 0; + + INIT_LIST_HEAD(&qlist); + + for_each_set_bit_inv(apid, apm_reset, AP_DEVICES) + collect_queues_to_reset(matrix_mdev, apid, &qlist); + + return vfio_ap_mdev_reset_qlist(&qlist); +} + /** * assign_adapter_store - parses the APID from @buf and sets the * corresponding bit in the mediated matrix device's APM @@ -958,10 +1076,10 @@ static ssize_t assign_adapter_store(struct device *dev, { int ret; unsigned long apid; - DECLARE_BITMAP(apm_delta, AP_DEVICES); + DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apid); @@ -987,17 +1105,16 @@ static ssize_t assign_adapter_store(struct device *dev, } vfio_ap_mdev_link_adapter(matrix_mdev, apid); - memset(apm_delta, 0, sizeof(apm_delta)); - set_bit_inv(apid, apm_delta); - if (vfio_ap_mdev_filter_matrix(apm_delta, - matrix_mdev->matrix.aqm, matrix_mdev)) + if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + } ret = count; done: release_update_locks_for_mdev(matrix_mdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return ret; } @@ -1023,11 +1140,12 @@ static struct vfio_ap_queue * adapter was assigned. * @matrix_mdev: the matrix mediated device to which the adapter was assigned. * @apid: the APID of the unassigned adapter. - * @qtable: table for storing queues associated with unassigned adapter. + * @qlist: list for storing queues associated with unassigned adapter that + * need to be reset. */ static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, unsigned long apid, - struct ap_queue_table *qtable) + struct list_head *qlist) { unsigned long apqi; struct vfio_ap_queue *q; @@ -1035,38 +1153,53 @@ static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); - if (q && qtable) { + if (q && qlist) { if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) - hash_add(qtable->queues, &q->mdev_qnode, - q->apqn); + list_add_tail(&q->reset_qnode, qlist); } } } -static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, - unsigned long apid) +static void vfio_ap_mdev_hot_unplug_adapters(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apids) { - int loop_cursor; - struct vfio_ap_queue *q; - struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); + struct vfio_ap_queue *q, *tmpq; + struct list_head qlist; + unsigned long apid; + bool apcb_update = false; - hash_init(qtable->queues); - vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable); + INIT_LIST_HEAD(&qlist); - if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { - clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); + for_each_set_bit_inv(apid, apids, AP_DEVICES) { + vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist); + + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + apcb_update = true; + } } - vfio_ap_mdev_reset_queues(qtable); + /* Only update apcb if needed to avoid impacting guest */ + if (apcb_update) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + + vfio_ap_mdev_reset_qlist(&qlist); - hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { + list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { vfio_ap_unlink_mdev_fr_queue(q); - hash_del(&q->mdev_qnode); + list_del(&q->reset_qnode); } +} + +static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + DECLARE_BITMAP(apids, AP_DEVICES); - kfree(qtable); + bitmap_zero(apids, AP_DEVICES); + set_bit_inv(apid, apids); + vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, apids); } /** @@ -1167,10 +1300,10 @@ static ssize_t assign_domain_store(struct device *dev, { int ret; unsigned long apqi; - DECLARE_BITMAP(aqm_delta, AP_DOMAINS); + DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apqi); @@ -1196,17 +1329,16 @@ static ssize_t assign_domain_store(struct device *dev, } vfio_ap_mdev_link_domain(matrix_mdev, apqi); - memset(aqm_delta, 0, sizeof(aqm_delta)); - set_bit_inv(apqi, aqm_delta); - if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, - matrix_mdev)) + if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + } ret = count; done: release_update_locks_for_mdev(matrix_mdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return ret; } @@ -1214,7 +1346,7 @@ static DEVICE_ATTR_WO(assign_domain); static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, unsigned long apqi, - struct ap_queue_table *qtable) + struct list_head *qlist) { unsigned long apid; struct vfio_ap_queue *q; @@ -1222,38 +1354,53 @@ static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); - if (q && qtable) { + if (q && qlist) { if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) - hash_add(qtable->queues, &q->mdev_qnode, - q->apqn); + list_add_tail(&q->reset_qnode, qlist); } } } -static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, - unsigned long apqi) +static void vfio_ap_mdev_hot_unplug_domains(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apqis) { - int loop_cursor; - struct vfio_ap_queue *q; - struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); + struct vfio_ap_queue *q, *tmpq; + struct list_head qlist; + unsigned long apqi; + bool apcb_update = false; - hash_init(qtable->queues); - vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable); + INIT_LIST_HEAD(&qlist); - if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { - clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); + for_each_set_bit_inv(apqi, apqis, AP_DOMAINS) { + vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist); + + if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { + clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); + apcb_update = true; + } } - vfio_ap_mdev_reset_queues(qtable); + /* Only update apcb if needed to avoid impacting guest */ + if (apcb_update) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); - hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { + vfio_ap_mdev_reset_qlist(&qlist); + + list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { vfio_ap_unlink_mdev_fr_queue(q); - hash_del(&q->mdev_qnode); + list_del(&q->reset_qnode); } +} + +static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, + unsigned long apqi) +{ + DECLARE_BITMAP(apqis, AP_DOMAINS); - kfree(qtable); + bitmap_zero(apqis, AP_DEVICES); + set_bit_inv(apqi, apqis); + vfio_ap_mdev_hot_unplug_domains(matrix_mdev, apqis); } /** @@ -1416,18 +1563,13 @@ static ssize_t control_domains_show(struct device *dev, char *buf) { unsigned long id; - int nchars = 0; - int n; - char *bufpos = buf; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_domid = matrix_mdev->matrix.adm_max; + int nchars = 0; mutex_lock(&matrix_dev->mdevs_lock); - for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) { - n = sprintf(bufpos, "%04lx\n", id); - bufpos += n; - nchars += n; - } + for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) + nchars += sysfs_emit_at(buf, nchars, "%04lx\n", id); mutex_unlock(&matrix_dev->mdevs_lock); return nchars; @@ -1436,7 +1578,6 @@ static DEVICE_ATTR_RO(control_domains); static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) { - char *bufpos = buf; unsigned long apid; unsigned long apqi; unsigned long apid1; @@ -1444,33 +1585,21 @@ static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) unsigned long napm_bits = matrix->apm_max + 1; unsigned long naqm_bits = matrix->aqm_max + 1; int nchars = 0; - int n; apid1 = find_first_bit_inv(matrix->apm, napm_bits); apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits); if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { for_each_set_bit_inv(apid, matrix->apm, napm_bits) { - for_each_set_bit_inv(apqi, matrix->aqm, - naqm_bits) { - n = sprintf(bufpos, "%02lx.%04lx\n", apid, - apqi); - bufpos += n; - nchars += n; - } + for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) + nchars += sysfs_emit_at(buf, nchars, "%02lx.%04lx\n", apid, apqi); } } else if (apid1 < napm_bits) { - for_each_set_bit_inv(apid, matrix->apm, napm_bits) { - n = sprintf(bufpos, "%02lx.\n", apid); - bufpos += n; - nchars += n; - } + for_each_set_bit_inv(apid, matrix->apm, napm_bits) + nchars += sysfs_emit_at(buf, nchars, "%02lx.\n", apid); } else if (apqi1 < naqm_bits) { - for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) { - n = sprintf(bufpos, ".%04lx\n", apqi); - bufpos += n; - nchars += n; - } + for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) + nchars += sysfs_emit_at(buf, nchars, ".%04lx\n", apqi); } return nchars; @@ -1504,6 +1633,158 @@ static ssize_t guest_matrix_show(struct device *dev, } static DEVICE_ATTR_RO(guest_matrix); +static ssize_t write_ap_bitmap(unsigned long *bitmap, char *buf, int offset, char sep) +{ + return sysfs_emit_at(buf, offset, "0x%016lx%016lx%016lx%016lx%c", + bitmap[0], bitmap[1], bitmap[2], bitmap[3], sep); +} + +static ssize_t ap_config_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + int idx = 0; + + idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ','); + idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ','); + idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n'); + + return idx; +} + +/* Number of characters needed for a complete hex mask representing the bits in .. */ +#define AP_DEVICES_STRLEN (AP_DEVICES / 4 + 3) +#define AP_DOMAINS_STRLEN (AP_DOMAINS / 4 + 3) +#define AP_CONFIG_STRLEN (AP_DEVICES_STRLEN + 2 * AP_DOMAINS_STRLEN) + +static int parse_bitmap(char **strbufptr, unsigned long *bitmap, int nbits) +{ + char *curmask; + + curmask = strsep(strbufptr, ",\n"); + if (!curmask) + return -EINVAL; + + bitmap_clear(bitmap, 0, nbits); + return ap_hex2bitmap(curmask, bitmap, nbits); +} + +static int ap_matrix_overflow_check(struct ap_matrix_mdev *matrix_mdev) +{ + unsigned long bit; + + for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) { + if (bit > matrix_mdev->matrix.apm_max) + return -ENODEV; + } + + for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) { + if (bit > matrix_mdev->matrix.aqm_max) + return -ENODEV; + } + + for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) { + if (bit > matrix_mdev->matrix.adm_max) + return -ENODEV; + } + + return 0; +} + +static void ap_matrix_copy(struct ap_matrix *dst, struct ap_matrix *src) +{ + /* This check works around false positive gcc -Wstringop-overread */ + if (!src) + return; + + bitmap_copy(dst->apm, src->apm, AP_DEVICES); + bitmap_copy(dst->aqm, src->aqm, AP_DOMAINS); + bitmap_copy(dst->adm, src->adm, AP_DOMAINS); +} + +static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + struct ap_matrix m_new, m_old, m_added, m_removed; + DECLARE_BITMAP(apm_filtered, AP_DEVICES); + unsigned long newbit; + char *newbuf, *rest; + int rc = count; + bool do_update; + + newbuf = kstrndup(buf, AP_CONFIG_STRLEN, GFP_KERNEL); + if (!newbuf) + return -ENOMEM; + rest = newbuf; + + mutex_lock(&ap_attr_mutex); + get_update_locks_for_mdev(matrix_mdev); + + /* Save old state */ + ap_matrix_copy(&m_old, &matrix_mdev->matrix); + if (parse_bitmap(&rest, m_new.apm, AP_DEVICES) || + parse_bitmap(&rest, m_new.aqm, AP_DOMAINS) || + parse_bitmap(&rest, m_new.adm, AP_DOMAINS)) { + rc = -EINVAL; + goto out; + } + + bitmap_andnot(m_removed.apm, m_old.apm, m_new.apm, AP_DEVICES); + bitmap_andnot(m_removed.aqm, m_old.aqm, m_new.aqm, AP_DOMAINS); + bitmap_andnot(m_added.apm, m_new.apm, m_old.apm, AP_DEVICES); + bitmap_andnot(m_added.aqm, m_new.aqm, m_old.aqm, AP_DOMAINS); + + /* Need new bitmaps in matrix_mdev for validation */ + ap_matrix_copy(&matrix_mdev->matrix, &m_new); + + /* Ensure new state is valid, else undo new state */ + rc = vfio_ap_mdev_validate_masks(matrix_mdev); + if (rc) { + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + goto out; + } + rc = ap_matrix_overflow_check(matrix_mdev); + if (rc) { + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + goto out; + } + rc = count; + + /* Need old bitmaps in matrix_mdev for unplug/unlink */ + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + + /* Unlink removed adapters/domains */ + vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, m_removed.apm); + vfio_ap_mdev_hot_unplug_domains(matrix_mdev, m_removed.aqm); + + /* Need new bitmaps in matrix_mdev for linking new adapters/domains */ + ap_matrix_copy(&matrix_mdev->matrix, &m_new); + + /* Link newly added adapters */ + for_each_set_bit_inv(newbit, m_added.apm, AP_DEVICES) + vfio_ap_mdev_link_adapter(matrix_mdev, newbit); + + for_each_set_bit_inv(newbit, m_added.aqm, AP_DOMAINS) + vfio_ap_mdev_link_domain(matrix_mdev, newbit); + + /* filter resources not bound to vfio-ap */ + do_update = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered); + do_update |= vfio_ap_mdev_filter_cdoms(matrix_mdev); + + /* Apply changes to shadow apbc if things changed */ + if (do_update) { + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + } +out: + release_update_locks_for_mdev(matrix_mdev); + mutex_unlock(&ap_attr_mutex); + kfree(newbuf); + return rc; +} +static DEVICE_ATTR_RW(ap_config); + static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_assign_adapter.attr, &dev_attr_unassign_adapter.attr, @@ -1511,6 +1792,7 @@ static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_unassign_domain.attr, &dev_attr_assign_control_domain.attr, &dev_attr_unassign_control_domain.attr, + &dev_attr_ap_config.attr, &dev_attr_control_domains.attr, &dev_attr_matrix.attr, &dev_attr_guest_matrix.attr, @@ -1608,7 +1890,7 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) get_update_locks_for_kvm(kvm); kvm_arch_crypto_clear_masks(kvm); - vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); + vfio_ap_mdev_reset_queues(matrix_mdev); kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; @@ -1638,6 +1920,7 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) switch (status->response_code) { case AP_RESPONSE_NORMAL: case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: return 0; case AP_RESPONSE_RESET_IN_PROGRESS: case AP_RESPONSE_BUSY: @@ -1694,14 +1977,6 @@ static void apq_reset_check(struct work_struct *reset_work) memcpy(&q->reset_status, &status, sizeof(status)); continue; } - /* - * When an AP adapter is deconfigured, the - * associated queues are reset, so let's set the - * status response code to 0 so the queue may be - * passed through (i.e., not filtered) - */ - if (status.response_code == AP_RESPONSE_DECONFIGURED) - q->reset_status.response_code = 0; if (q->saved_isc != VFIO_AP_ISC_INVALID) vfio_ap_free_aqic_resources(q); break; @@ -1728,12 +2003,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) queue_work(system_long_wq, &q->reset_work); break; case AP_RESPONSE_DECONFIGURED: - /* - * When an AP adapter is deconfigured, the associated - * queues are reset, so let's set the status response code to 0 - * so the queue may be passed through (i.e., not filtered). - */ - q->reset_status.response_code = 0; + case AP_RESPONSE_CHECKSTOPPED: vfio_ap_free_aqic_resources(q); break; default: @@ -1744,15 +2014,33 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) } } -static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable) +static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev) { int ret = 0, loop_cursor; struct vfio_ap_queue *q; - hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) + hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) vfio_ap_mdev_reset_queue(q); - hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { + hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) { + flush_work(&q->reset_work); + + if (q->reset_status.response_code) + ret = -EIO; + } + + return ret; +} + +static int vfio_ap_mdev_reset_qlist(struct list_head *qlist) +{ + int ret = 0; + struct vfio_ap_queue *q; + + list_for_each_entry(q, qlist, reset_qnode) + vfio_ap_mdev_reset_queue(q); + + list_for_each_entry(q, qlist, reset_qnode) { flush_work(&q->reset_work); if (q->reset_status.response_code) @@ -1788,17 +2076,26 @@ static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count) matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev); + get_update_locks_for_mdev(matrix_mdev); + + if (matrix_mdev->kvm) { + kvm_arch_crypto_clear_masks(matrix_mdev->kvm); + signal_guest_ap_cfg_changed(matrix_mdev); + } + if (matrix_mdev->req_trigger) { if (!(count % 10)) dev_notice_ratelimited(dev, "Relaying device request to user (#%u)\n", count); - eventfd_signal(matrix_mdev->req_trigger, 1); + eventfd_signal(matrix_mdev->req_trigger); } else if (count == 0) { dev_notice(dev, "No device request registered, blocked until released by user\n"); } + + release_update_locks_for_mdev(matrix_mdev); } static int vfio_ap_mdev_get_device_info(unsigned long arg) @@ -1839,6 +2136,10 @@ static ssize_t vfio_ap_get_irq_info(unsigned long arg) info.count = 1; info.flags = VFIO_IRQ_INFO_EVENTFD; break; + case VFIO_AP_CFG_CHG_IRQ_INDEX: + info.count = 1; + info.flags = VFIO_IRQ_INFO_EVENTFD; + break; default: return -EINVAL; } @@ -1902,6 +2203,39 @@ static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev, return 0; } +static int vfio_ap_set_cfg_change_irq(struct ap_matrix_mdev *matrix_mdev, unsigned long arg) +{ + s32 fd; + void __user *data; + unsigned long minsz; + struct eventfd_ctx *cfg_chg_trigger; + + minsz = offsetofend(struct vfio_irq_set, count); + data = (void __user *)(arg + minsz); + + if (get_user(fd, (s32 __user *)data)) + return -EFAULT; + + if (fd == -1) { + if (matrix_mdev->cfg_chg_trigger) + eventfd_ctx_put(matrix_mdev->cfg_chg_trigger); + matrix_mdev->cfg_chg_trigger = NULL; + } else if (fd >= 0) { + cfg_chg_trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(cfg_chg_trigger)) + return PTR_ERR(cfg_chg_trigger); + + if (matrix_mdev->cfg_chg_trigger) + eventfd_ctx_put(matrix_mdev->cfg_chg_trigger); + + matrix_mdev->cfg_chg_trigger = cfg_chg_trigger; + } else { + return -EINVAL; + } + + return 0; +} + static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev, unsigned long arg) { @@ -1917,6 +2251,8 @@ static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev, switch (irq_set.index) { case VFIO_AP_REQ_IRQ_INDEX: return vfio_ap_set_request_irq(matrix_mdev, arg); + case VFIO_AP_CFG_CHG_IRQ_INDEX: + return vfio_ap_set_cfg_change_irq(matrix_mdev, arg); default: return -EINVAL; } @@ -1938,11 +2274,11 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, ret = vfio_ap_mdev_get_device_info(arg); break; case VFIO_DEVICE_RESET: - ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); + ret = vfio_ap_mdev_reset_queues(matrix_mdev); break; case VFIO_DEVICE_GET_IRQ_INFO: - ret = vfio_ap_get_irq_info(arg); - break; + ret = vfio_ap_get_irq_info(arg); + break; case VFIO_DEVICE_SET_IRQS: ret = vfio_ap_set_irqs(matrix_mdev, arg); break; @@ -1976,6 +2312,7 @@ static ssize_t status_show(struct device *dev, { ssize_t nchars = 0; struct vfio_ap_queue *q; + unsigned long apid, apqi; struct ap_matrix_mdev *matrix_mdev; struct ap_device *apdev = to_ap_dev(dev); @@ -1983,16 +2320,26 @@ static ssize_t status_show(struct device *dev, q = dev_get_drvdata(&apdev->device); matrix_mdev = vfio_ap_mdev_for_queue(q); + /* If the queue is assigned to the matrix mediated device, then + * determine whether it is passed through to a guest; otherwise, + * indicate that it is unassigned. + */ if (matrix_mdev) { - if (matrix_mdev->kvm) - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_IN_USE); + apid = AP_QID_CARD(q->apqn); + apqi = AP_QID_QUEUE(q->apqn); + /* + * If the queue is passed through to the guest, then indicate + * that it is in use; otherwise, indicate that it is + * merely assigned to a matrix mediated device. + */ + if (matrix_mdev->kvm && + test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && + test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_IN_USE); else - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_ASSIGNED); + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_ASSIGNED); } else { - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_UNASSIGNED); + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); } mutex_unlock(&matrix_dev->mdevs_lock); @@ -2020,6 +2367,7 @@ static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { .bind_iommufd = vfio_iommufd_emulated_bind, .unbind_iommufd = vfio_iommufd_emulated_unbind, .attach_ioas = vfio_iommufd_emulated_attach_ioas, + .detach_ioas = vfio_iommufd_emulated_detach_ioas, .request = vfio_ap_mdev_request }; @@ -2046,10 +2394,10 @@ int vfio_ap_mdev_register(void) matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT; matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT; - matrix_dev->mdev_types[0] = &matrix_dev->mdev_type; + matrix_dev->mdev_types = &matrix_dev->mdev_type; ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device, &vfio_ap_matrix_driver, - matrix_dev->mdev_types, 1); + &matrix_dev->mdev_types, 1); if (ret) goto err_driver; return 0; @@ -2069,6 +2417,7 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { int ret; struct vfio_ap_queue *q; + DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev; ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); @@ -2090,15 +2439,28 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) if (matrix_mdev) { vfio_ap_mdev_link_queue(matrix_mdev, q); - if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, - matrix_mdev->matrix.aqm, - matrix_mdev)) + /* + * If we're in the process of handling the adding of adapters or + * domains to the host's AP configuration, then let the + * vfio_ap device driver's on_scan_complete callback filter the + * matrix and update the guest's AP configuration after all of + * the new queue devices are probed. + */ + if (!bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) || + !bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS)) + goto done; + + if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) { vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + } } + +done: dev_set_drvdata(&apdev->device, q); release_update_locks_for_mdev(matrix_mdev); - return 0; + return ret; err_remove_group: sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); @@ -2115,26 +2477,40 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) q = dev_get_drvdata(&apdev->device); get_update_locks_for_queue(q); matrix_mdev = q->matrix_mdev; + apid = AP_QID_CARD(q->apqn); + apqi = AP_QID_QUEUE(q->apqn); if (matrix_mdev) { - vfio_ap_unlink_queue_fr_mdev(q); - - apid = AP_QID_CARD(q->apqn); - apqi = AP_QID_QUEUE(q->apqn); - - /* - * If the queue is assigned to the guest's APCB, then remove - * the adapter's APID from the APCB and hot it into the guest. - */ + /* If the queue is assigned to the guest's AP configuration */ if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { + /* + * Since the queues are defined via a matrix of adapters + * and domains, it is not possible to hot unplug a + * single queue; so, let's unplug the adapter. + */ clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apid(matrix_mdev, apid); + goto done; } } - vfio_ap_mdev_reset_queue(q); - flush_work(&q->reset_work); + /* + * If the queue is not in the host's AP configuration, then resetting + * it will fail with response code 01, (APQN not valid); so, let's make + * sure it is in the host's config. + */ + if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) && + test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) { + vfio_ap_mdev_reset_queue(q); + flush_work(&q->reset_work); + } + +done: + if (matrix_mdev) + vfio_ap_unlink_queue_fr_mdev(q); + dev_set_drvdata(&apdev->device, NULL); kfree(q); release_update_locks_for_mdev(matrix_mdev); @@ -2160,7 +2536,7 @@ int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); - ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); + ret = vfio_ap_mdev_verify_no_sharing(NULL, apm, aqm); mutex_unlock(&matrix_dev->mdevs_lock); mutex_unlock(&matrix_dev->guests_lock); @@ -2296,7 +2672,7 @@ static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm) bool apid_cleared; struct ap_queue_status status; unsigned long apid, apqi; - struct ap_tapq_gr2 info; + struct ap_tapq_hwinfo info; for_each_set_bit_inv(apid, apm, AP_DEVICES) { apid_cleared = false; @@ -2442,39 +2818,30 @@ void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) { - bool do_hotplug = false; - int filter_domains = 0; - int filter_adapters = 0; - DECLARE_BITMAP(apm, AP_DEVICES); - DECLARE_BITMAP(aqm, AP_DOMAINS); + DECLARE_BITMAP(apm_filtered, AP_DEVICES); + bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; mutex_lock(&matrix_mdev->kvm->lock); mutex_lock(&matrix_dev->mdevs_lock); - filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm, - matrix_mdev->apm_add, AP_DEVICES); - filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm, - matrix_mdev->aqm_add, AP_DOMAINS); - - if (filter_adapters && filter_domains) - do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev); - else if (filter_adapters) - do_hotplug |= - vfio_ap_mdev_filter_matrix(apm, - matrix_mdev->shadow_apcb.aqm, - matrix_mdev); - else - do_hotplug |= - vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm, - aqm, matrix_mdev); + filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, + matrix_mdev->apm_add, AP_DEVICES); + filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm, + matrix_mdev->aqm_add, AP_DOMAINS); + filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm, + matrix_mdev->adm_add, AP_DOMAINS); - if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add, - AP_DOMAINS)) + if (filter_adapters || filter_domains) + do_hotplug = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered); + + if (filter_cdoms) do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev); if (do_hotplug) vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + mutex_unlock(&matrix_dev->mdevs_lock); mutex_unlock(&matrix_mdev->kvm->lock); } diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 88aff8b81f2f..9bff666b0b35 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -53,7 +53,7 @@ struct ap_matrix_dev { struct mutex guests_lock; /* serializes access to each KVM guest */ struct mdev_parent parent; struct mdev_type mdev_type; - struct mdev_type *mdev_types[1]; + struct mdev_type *mdev_types; }; extern struct ap_matrix_dev *matrix_dev; @@ -75,11 +75,11 @@ extern struct ap_matrix_dev *matrix_dev; */ struct ap_matrix { unsigned long apm_max; - DECLARE_BITMAP(apm, 256); + DECLARE_BITMAP(apm, AP_DEVICES); unsigned long aqm_max; - DECLARE_BITMAP(aqm, 256); + DECLARE_BITMAP(aqm, AP_DOMAINS); unsigned long adm_max; - DECLARE_BITMAP(adm, 256); + DECLARE_BITMAP(adm, AP_DOMAINS); }; /** @@ -105,6 +105,7 @@ struct ap_queue_table { * @mdev: the mediated device * @qtable: table of queues (struct vfio_ap_queue) assigned to the mdev * @req_trigger eventfd ctx for signaling userspace to return a device + * @cfg_chg_trigger eventfd ctx to signal AP config changed to userspace * @apm_add: bitmap of APIDs added to the host's AP configuration * @aqm_add: bitmap of APQIs added to the host's AP configuration * @adm_add: bitmap of control domain numbers added to the host's AP @@ -120,6 +121,7 @@ struct ap_matrix_mdev { struct mdev_device *mdev; struct ap_queue_table qtable; struct eventfd_ctx *req_trigger; + struct eventfd_ctx *cfg_chg_trigger; DECLARE_BITMAP(apm_add, AP_DEVICES); DECLARE_BITMAP(aqm_add, AP_DOMAINS); DECLARE_BITMAP(adm_add, AP_DOMAINS); @@ -133,6 +135,8 @@ struct ap_matrix_mdev { * @apqn: the APQN of the AP queue device * @saved_isc: the guest ISC registered with the GIB interface * @mdev_qnode: allows the vfio_ap_queue struct to be added to a hashtable + * @reset_qnode: allows the vfio_ap_queue struct to be added to a list of queues + * that need to be reset * @reset_status: the status from the last reset of the queue * @reset_work: work to wait for queue reset to complete */ @@ -143,6 +147,7 @@ struct vfio_ap_queue { #define VFIO_AP_ISC_INVALID 0xff unsigned char saved_isc; struct hlist_node mdev_qnode; + struct list_head reset_qnode; struct ap_queue_status reset_status; struct work_struct reset_work; }; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 4b23c9f7f3e5..7a3b99f065f2 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -12,12 +12,14 @@ * Multiple device nodes: Harald Freudenberger <freude@linux.ibm.com> */ +#define pr_fmt(fmt) "zcrypt: " fmt + +#include <linux/export.h> #include <linux/module.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/miscdevice.h> #include <linux/fs.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> @@ -47,6 +49,10 @@ MODULE_DESCRIPTION("Cryptographic Coprocessor interface, " \ "Copyright IBM Corp. 2001, 2012"); MODULE_LICENSE("GPL"); +unsigned int zcrypt_mempool_threshold = 5; +module_param_named(mempool_threshold, zcrypt_mempool_threshold, uint, 0440); +MODULE_PARM_DESC(mempool_threshold, "CCA and EP11 request/reply mempool minimal items (min: 1)"); + /* * zcrypt tracepoint functions */ @@ -57,10 +63,6 @@ DEFINE_SPINLOCK(zcrypt_list_lock); LIST_HEAD(zcrypt_card_list); static atomic_t zcrypt_open_count = ATOMIC_INIT(0); -static atomic_t zcrypt_rescan_count = ATOMIC_INIT(0); - -atomic_t zcrypt_rescan_req = ATOMIC_INIT(0); -EXPORT_SYMBOL(zcrypt_rescan_req); static LIST_HEAD(zcrypt_ops_list); @@ -69,20 +71,15 @@ debug_info_t *zcrypt_dbf_info; /* * Process a rescan of the transport layer. - * - * Returns 1, if the rescan has been processed, otherwise 0. + * Runs a synchronous AP bus rescan. + * Returns true if something has changed (for example the + * bus scan has found and build up new devices) and it is + * worth to do a retry. Otherwise false is returned meaning + * no changes on the AP bus level. */ -static inline int zcrypt_process_rescan(void) +static inline bool zcrypt_process_rescan(void) { - if (atomic_read(&zcrypt_rescan_req)) { - atomic_set(&zcrypt_rescan_req, 0); - atomic_inc(&zcrypt_rescan_count); - ap_bus_force_rescan(); - ZCRYPT_DBF_INFO("%s rescan count=%07d\n", __func__, - atomic_inc_return(&zcrypt_rescan_count)); - return 1; - } - return 0; + return ap_bus_force_rescan(); } void zcrypt_msgtype_register(struct zcrypt_ops *zops) @@ -113,7 +110,11 @@ EXPORT_SYMBOL(zcrypt_msgtype); struct zcdn_device; -static struct class *zcrypt_class; +static void zcdn_device_release(struct device *dev); +static const struct class zcrypt_class = { + .name = ZCRYPT_NAME, + .dev_release = zcdn_device_release, +}; static dev_t zcrypt_devt; static struct cdev zcrypt_cdev; @@ -136,7 +137,7 @@ static int zcdn_destroy(const char *name); */ static inline struct zcdn_device *find_zcdndev_by_name(const char *name) { - struct device *dev = class_find_device_by_name(zcrypt_class, name); + struct device *dev = class_find_device_by_name(&zcrypt_class, name); return dev ? to_zcdn_dev(dev) : NULL; } @@ -148,7 +149,7 @@ static inline struct zcdn_device *find_zcdndev_by_name(const char *name) */ static inline struct zcdn_device *find_zcdndev_by_devt(dev_t devt) { - struct device *dev = class_find_device_by_devt(zcrypt_class, devt); + struct device *dev = class_find_device_by_devt(&zcrypt_class, devt); return dev ? to_zcdn_dev(dev) : NULL; } @@ -160,7 +161,7 @@ static ssize_t ioctlmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -168,7 +169,7 @@ static ssize_t ioctlmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.ioctlm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -181,7 +182,7 @@ static ssize_t ioctlmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.ioctlm, - AP_IOCTLS, &ap_perms_mutex); + AP_IOCTLS, &ap_attr_mutex); if (rc) return rc; @@ -197,7 +198,7 @@ static ssize_t apmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -205,7 +206,7 @@ static ssize_t apmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.apm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -218,7 +219,7 @@ static ssize_t apmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.apm, - AP_DEVICES, &ap_perms_mutex); + AP_DEVICES, &ap_attr_mutex); if (rc) return rc; @@ -234,7 +235,7 @@ static ssize_t aqmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -242,7 +243,7 @@ static ssize_t aqmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.aqm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -255,7 +256,7 @@ static ssize_t aqmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.aqm, - AP_DOMAINS, &ap_perms_mutex); + AP_DOMAINS, &ap_attr_mutex); if (rc) return rc; @@ -271,7 +272,7 @@ static ssize_t admask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -279,7 +280,7 @@ static ssize_t admask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.adm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -292,7 +293,7 @@ static ssize_t admask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.adm, - AP_DOMAINS, &ap_perms_mutex); + AP_DOMAINS, &ap_attr_mutex); if (rc) return rc; @@ -366,10 +367,9 @@ static int zcdn_create(const char *name) { dev_t devt; int i, rc = 0; - char nodename[ZCDN_MAX_NAME]; struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; /* check if device node with this name already exists */ @@ -403,17 +403,15 @@ static int zcdn_create(const char *name) goto unlockout; } zcdndev->device.release = zcdn_device_release; - zcdndev->device.class = zcrypt_class; + zcdndev->device.class = &zcrypt_class; zcdndev->device.devt = devt; zcdndev->device.groups = zcdn_dev_attr_groups; if (name[0]) - strncpy(nodename, name, sizeof(nodename)); + rc = dev_set_name(&zcdndev->device, "%s", name); else - snprintf(nodename, sizeof(nodename), - ZCRYPT_NAME "_%d", (int)MINOR(devt)); - nodename[sizeof(nodename) - 1] = '\0'; - if (dev_set_name(&zcdndev->device, nodename)) { - rc = -EINVAL; + rc = dev_set_name(&zcdndev->device, ZCRYPT_NAME "_%d", (int)MINOR(devt)); + if (rc) { + kfree(zcdndev); goto unlockout; } rc = device_register(&zcdndev->device); @@ -426,7 +424,7 @@ static int zcdn_create(const char *name) __func__, MAJOR(devt), MINOR(devt)); unlockout: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -435,7 +433,7 @@ static int zcdn_destroy(const char *name) int rc = 0; struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; /* try to find this zcdn device */ @@ -453,7 +451,7 @@ static int zcdn_destroy(const char *name) device_unregister(&zcdndev->device); unlockout: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -463,7 +461,7 @@ static void zcdn_destroy_all(void) dev_t devt; struct zcdn_device *zcdndev; - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) { devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i); zcdndev = find_zcdndev_by_devt(devt); @@ -472,7 +470,7 @@ static void zcdn_destroy_all(void) device_unregister(&zcdndev->device); } } - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); } /* @@ -509,11 +507,11 @@ static int zcrypt_open(struct inode *inode, struct file *filp) if (filp->f_inode->i_cdev == &zcrypt_cdev) { struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev); /* find returns a reference, no get_device() needed */ - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (zcdndev) perms = &zcdndev->perms; } @@ -533,9 +531,9 @@ static int zcrypt_release(struct inode *inode, struct file *filp) if (filp->f_inode->i_cdev == &zcrypt_cdev) { struct zcdn_device *zcdndev; - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (zcdndev) { /* 2 puts here: one for find, one for open */ put_device(&zcdndev->device); @@ -582,6 +580,7 @@ static inline struct zcrypt_queue *zcrypt_pick_queue(struct zcrypt_card *zc, { if (!zq || !try_module_get(zq->queue->ap_dev.device.driver->owner)) return NULL; + zcrypt_card_get(zc); zcrypt_queue_get(zq); get_device(&zq->queue->ap_dev.device); atomic_add(weight, &zc->load); @@ -601,6 +600,7 @@ static inline void zcrypt_drop_queue(struct zcrypt_card *zc, atomic_sub(weight, &zq->load); put_device(&zq->queue->ap_dev.device); zcrypt_queue_put(zq); + zcrypt_card_put(zc); module_put(mod); } @@ -645,16 +645,17 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, struct zcrypt_queue *zq, *pref_zq; struct ap_message ap_msg; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code; - int cpen, qpen, qid = 0, rc = -ENODEV; + unsigned int func_code = 0; + int cpen, qpen, qid = 0, rc; struct module *mod; trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO); - ap_init_message(&ap_msg); + rc = ap_init_apmsg(&ap_msg, 0); + if (rc) + goto out; if (mex->outputdatalength < mex->inputdatalength) { - func_code = 0; rc = -EINVAL; goto out; } @@ -676,7 +677,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, for_each_zcrypt_card(zc) { /* Check for usable accelerator or CCA card */ if (!zc->online || !zc->card->config || zc->card->chkstop || - !(zc->card->functions & 0x18000000)) + !(zc->card->hwinfo.accel || zc->card->hwinfo.cca)) continue; /* Check for size limits */ if (zc->min_mod_size > mex->inputdatalength || @@ -696,7 +697,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -718,8 +719,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -732,13 +732,14 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); out: - ap_release_message(&ap_msg); + ap_release_apmsg(&ap_msg); if (tr) { tr->last_rc = rc; tr->last_qid = qid; } trace_s390_zcrypt_rep(mex, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -750,16 +751,17 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, struct zcrypt_queue *zq, *pref_zq; struct ap_message ap_msg; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code; - int cpen, qpen, qid = 0, rc = -ENODEV; + unsigned int func_code = 0; + int cpen, qpen, qid = 0, rc; struct module *mod; trace_s390_zcrypt_req(crt, TP_ICARSACRT); - ap_init_message(&ap_msg); + rc = ap_init_apmsg(&ap_msg, 0); + if (rc) + goto out; if (crt->outputdatalength < crt->inputdatalength) { - func_code = 0; rc = -EINVAL; goto out; } @@ -781,7 +783,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, for_each_zcrypt_card(zc) { /* Check for usable accelerator or CCA card */ if (!zc->online || !zc->card->config || zc->card->chkstop || - !(zc->card->functions & 0x18000000)) + !(zc->card->hwinfo.accel || zc->card->hwinfo.cca)) continue; /* Check for size limits */ if (zc->min_mod_size > crt->inputdatalength || @@ -801,7 +803,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo_crt || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -823,8 +825,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -837,37 +838,45 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); out: - ap_release_message(&ap_msg); + ap_release_apmsg(&ap_msg); if (tr) { tr->last_rc = rc; tr->last_qid = qid; } trace_s390_zcrypt_rep(crt, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } -static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, +static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, struct zcrypt_track *tr, struct ica_xcRB *xcrb) { + bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE; struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; struct ap_message ap_msg; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code; + unsigned int func_code = 0; unsigned short *domain, tdom; - int cpen, qpen, qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc; struct module *mod; trace_s390_zcrypt_req(xcrb, TB_ZSECSENDCPRB); xcrb->status = 0; - ap_init_message(&ap_msg); + + rc = ap_init_apmsg(&ap_msg, xflags & ZCRYPT_XFLAG_NOMEMALLOC ? + AP_MSG_FLAG_MEMPOOL : 0); + if (rc) + goto out; rc = prep_cca_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) goto out; + print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); tdom = *domain; if (perms != &ap_perms && tdom < AP_DOMAINS) { @@ -896,7 +905,7 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_card(zc) { /* Check for usable CCA card */ if (!zc->online || !zc->card->config || zc->card->chkstop || - !(zc->card->functions & 0x10000000)) + !zc->card->hwinfo.cca) continue; /* Check for user selected CCA card */ if (xcrb->user_defined != AUTOSELECT && @@ -919,7 +928,7 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check for device usable and eligible */ if (!zq->online || !zq->ops->send_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (tdom != AUTOSEL_DOM && tdom != AP_QID_QUEUE(zq->queue->qid))) continue; @@ -943,8 +952,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, xcrb->user_defined, *domain); + pr_debug("no match for address %02x.%04x => ENODEV\n", + xcrb->user_defined, *domain); rc = -ENODEV; goto out; } @@ -955,25 +964,49 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, *domain = AP_QID_QUEUE(qid); rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ccarpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out: - ap_release_message(&ap_msg); + ap_release_apmsg(&ap_msg); if (tr) { tr->last_rc = rc; tr->last_qid = qid; } trace_s390_zcrypt_rep(xcrb, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } -long zcrypt_send_cprb(struct ica_xcRB *xcrb) +long zcrypt_send_cprb(struct ica_xcRB *xcrb, u32 xflags) { - return _zcrypt_send_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_cprb(xflags, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_cprb(xflags, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("rc=%d\n", rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_cprb); @@ -1004,60 +1037,62 @@ static bool is_desired_ep11_queue(unsigned int dev_qid, return false; } -static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, +static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, struct zcrypt_track *tr, struct ep11_urb *xcrb) { + bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE; struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; - struct ep11_target_dev *targets; + struct ep11_target_dev *targets = NULL; unsigned short target_num; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code, domain; + unsigned int func_code = 0, domain; struct ap_message ap_msg; - int cpen, qpen, qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc; struct module *mod; trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB); - ap_init_message(&ap_msg); + rc = ap_init_apmsg(&ap_msg, xflags & ZCRYPT_XFLAG_NOMEMALLOC ? + AP_MSG_FLAG_MEMPOOL : 0); + if (rc) + goto out; target_num = (unsigned short)xcrb->targets_num; /* empty list indicates autoselect (all available targets) */ - targets = NULL; + rc = -ENOMEM; if (target_num != 0) { - struct ep11_target_dev __user *uptr; - - targets = kcalloc(target_num, sizeof(*targets), GFP_KERNEL); - if (!targets) { - func_code = 0; - rc = -ENOMEM; - goto out; - } - - uptr = (struct ep11_target_dev __force __user *)xcrb->targets; - if (z_copy_from_user(userspace, targets, uptr, - target_num * sizeof(*targets))) { - func_code = 0; - rc = -EFAULT; - goto out_free; + if (userspace) { + targets = kcalloc(target_num, sizeof(*targets), GFP_KERNEL); + if (!targets) + goto out; + if (copy_from_user(targets, xcrb->targets, + target_num * sizeof(*targets))) { + rc = -EFAULT; + goto out; + } + } else { + targets = (struct ep11_target_dev __force __kernel *)xcrb->targets; } } rc = prep_ep11_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) - goto out_free; + goto out; + print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); if (perms != &ap_perms && domain < AUTOSEL_DOM) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { if (!test_bit_inv(domain, perms->adm)) { rc = -ENODEV; - goto out_free; + goto out; } } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) { rc = -EOPNOTSUPP; - goto out_free; + goto out; } } @@ -1067,7 +1102,7 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_card(zc) { /* Check for usable EP11 card */ if (!zc->online || !zc->card->config || zc->card->chkstop || - !(zc->card->functions & 0x04000000)) + !zc->card->hwinfo.ep11) continue; /* Check for user selected EP11 card */ if (targets && @@ -1090,7 +1125,7 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->send_ep11_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (targets && !is_desired_ep11_queue(zq->queue->qid, target_num, targets))) @@ -1116,43 +1151,65 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (!pref_zq) { if (targets && target_num == 1) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, (int)targets->ap_id, - (int)targets->dom_id); + pr_debug("no match for address %02x.%04x => ENODEV\n", + (int)targets->ap_id, (int)targets->dom_id); } else if (targets) { - ZCRYPT_DBF_DBG("%s no match for %d target addrs => ENODEV\n", - __func__, (int)target_num); + pr_debug("no match for %d target addrs => ENODEV\n", + (int)target_num); } else { - ZCRYPT_DBF_DBG("%s no match for address ff.ffff => ENODEV\n", - __func__); + pr_debug("no match for address ff.ffff => ENODEV\n"); } rc = -ENODEV; - goto out_free; + goto out; } qid = pref_zq->queue->qid; rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ep11rpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); -out_free: - kfree(targets); out: - ap_release_message(&ap_msg); + if (userspace) + kfree(targets); + ap_release_apmsg(&ap_msg); if (tr) { tr->last_rc = rc; tr->last_qid = qid; } trace_s390_zcrypt_rep(xcrb, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } -long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) +long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb, u32 xflags) { - return _zcrypt_send_ep11_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_ep11_cprb(xflags, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_ep11_cprb(xflags, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("rc=%d\n", rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_ep11_cprb); @@ -1161,7 +1218,7 @@ static long zcrypt_rng(char *buffer) struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code; + unsigned int func_code = 0; struct ap_message ap_msg; unsigned int domain; int qid = 0, rc = -ENODEV; @@ -1169,7 +1226,9 @@ static long zcrypt_rng(char *buffer) trace_s390_zcrypt_req(buffer, TP_HWRNGCPRB); - ap_init_message(&ap_msg); + rc = ap_init_apmsg(&ap_msg, 0); + if (rc) + goto out; rc = prep_rng_ap_msg(&ap_msg, &func_code, &domain); if (rc) goto out; @@ -1180,7 +1239,7 @@ static long zcrypt_rng(char *buffer) for_each_zcrypt_card(zc) { /* Check for usable CCA card */ if (!zc->online || !zc->card->config || zc->card->chkstop || - !(zc->card->functions & 0x10000000)) + !zc->card->hwinfo.cca) continue; /* get weight index of the card device */ wgt = zc->speed_rating[func_code]; @@ -1189,7 +1248,7 @@ static long zcrypt_rng(char *buffer) for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rng || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt)) continue; @@ -1202,8 +1261,7 @@ static long zcrypt_rng(char *buffer) spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -1216,9 +1274,10 @@ static long zcrypt_rng(char *buffer) spin_unlock(&zcrypt_list_lock); out: - ap_release_message(&ap_msg); + ap_release_apmsg(&ap_msg); trace_s390_zcrypt_rep(buffer, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -1241,7 +1300,7 @@ static void zcrypt_device_status_mask(struct zcrypt_device_status *devstatus) queue = AP_QID_QUEUE(zq->queue->qid); stat = &devstatus[card * AP_DOMAINS + queue]; stat->hwtype = zc->card->ap_dev.device_type; - stat->functions = zc->card->functions >> 26; + stat->functions = zc->card->hwinfo.fac >> 26; stat->qid = zq->queue->qid; stat->online = zq->online ? 0x01 : 0x00; } @@ -1249,24 +1308,27 @@ static void zcrypt_device_status_mask(struct zcrypt_device_status *devstatus) spin_unlock(&zcrypt_list_lock); } -void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus) +void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus, + int maxcard, int maxqueue) { struct zcrypt_card *zc; struct zcrypt_queue *zq; struct zcrypt_device_status_ext *stat; int card, queue; - memset(devstatus, 0, MAX_ZDEV_ENTRIES_EXT - * sizeof(struct zcrypt_device_status_ext)); + maxcard = min_t(int, maxcard, MAX_ZDEV_CARDIDS_EXT); + maxqueue = min_t(int, maxqueue, MAX_ZDEV_DOMAINS_EXT); spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { for_each_zcrypt_queue(zq, zc) { card = AP_QID_CARD(zq->queue->qid); queue = AP_QID_QUEUE(zq->queue->qid); - stat = &devstatus[card * AP_DOMAINS + queue]; + if (card >= maxcard || queue >= maxqueue) + continue; + stat = &devstatus[card * maxqueue + queue]; stat->hwtype = zc->card->ap_dev.device_type; - stat->functions = zc->card->functions >> 26; + stat->functions = zc->card->hwinfo.fac >> 26; stat->qid = zq->queue->qid; stat->online = zq->online ? 0x01 : 0x00; } @@ -1289,7 +1351,7 @@ int zcrypt_device_status_ext(int card, int queue, if (card == AP_QID_CARD(zq->queue->qid) && queue == AP_QID_QUEUE(zq->queue->qid)) { devstat->hwtype = zc->card->ap_dev.device_type; - devstat->functions = zc->card->functions >> 26; + devstat->functions = zc->card->hwinfo.fac >> 26; devstat->qid = zq->queue->qid; devstat->online = zq->online ? 0x01 : 0x00; spin_unlock(&zcrypt_list_lock); @@ -1434,20 +1496,17 @@ static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSAMODEXPO rc=%d\n", rc); + pr_debug("ioctl ICARSAMODEXPO rc=%d\n", rc); return rc; } return put_user(mex.outputdatalength, &umex->outputdatalength); @@ -1466,20 +1525,17 @@ static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSACRT rc=%d\n", rc); + pr_debug("ioctl ICARSACRT rc=%d\n", rc); return rc; } return put_user(crt.outputdatalength, &ucrt->outputdatalength); @@ -1490,6 +1546,7 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) int rc; struct ica_xcRB xcrb; struct zcrypt_track tr; + u32 xflags = ZCRYPT_XFLAG_USERSPACE; struct ica_xcRB __user *uxcrb = (void __user *)arg; memset(&tr, 0, sizeof(tr)); @@ -1497,22 +1554,19 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) return -EFAULT; do { - rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { - rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDCPRB rc=%d status=0x%x\n", - rc, xcrb.status); + pr_debug("ioctl ZSENDCPRB rc=%d status=0x%x\n", + rc, xcrb.status); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1523,6 +1577,7 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) int rc; struct ep11_urb xcrb; struct zcrypt_track tr; + u32 xflags = ZCRYPT_XFLAG_USERSPACE; struct ep11_urb __user *uxcrb = (void __user *)arg; memset(&tr, 0, sizeof(tr)); @@ -1530,21 +1585,18 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) return -EFAULT; do { - rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + rc = _zcrypt_send_ep11_cprb(xflags, perms, &tr, &xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { - rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + rc = _zcrypt_send_ep11_cprb(xflags, perms, &tr, &xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDEP11CPRB rc=%d\n", rc); + pr_debug("ioctl ZSENDEP11CPRB rc=%d\n", rc); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1575,12 +1627,14 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, size_t total_size = MAX_ZDEV_ENTRIES_EXT * sizeof(struct zcrypt_device_status_ext); - device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT, - sizeof(struct zcrypt_device_status_ext), - GFP_KERNEL); + device_status = kvcalloc(MAX_ZDEV_ENTRIES_EXT, + sizeof(struct zcrypt_device_status_ext), + GFP_KERNEL); if (!device_status) return -ENOMEM; - zcrypt_device_status_mask_ext(device_status); + zcrypt_device_status_mask_ext(device_status, + MAX_ZDEV_CARDIDS_EXT, + MAX_ZDEV_DOMAINS_EXT); if (copy_to_user((char __user *)arg, device_status, total_size)) rc = -EFAULT; @@ -1673,210 +1727,11 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, } /* unknown ioctl number */ default: - ZCRYPT_DBF_DBG("unknown ioctl 0x%08x\n", cmd); + pr_debug("unknown ioctl 0x%08x\n", cmd); return -ENOIOCTLCMD; } } -#ifdef CONFIG_COMPAT -/* - * ioctl32 conversion routines - */ -struct compat_ica_rsa_modexpo { - compat_uptr_t inputdata; - unsigned int inputdatalength; - compat_uptr_t outputdata; - unsigned int outputdatalength; - compat_uptr_t b_key; - compat_uptr_t n_modulus; -}; - -static long trans_modexpo32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_rsa_modexpo __user *umex32 = compat_ptr(arg); - struct compat_ica_rsa_modexpo mex32; - struct ica_rsa_modexpo mex64; - struct zcrypt_track tr; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&mex32, umex32, sizeof(mex32))) - return -EFAULT; - mex64.inputdata = compat_ptr(mex32.inputdata); - mex64.inputdatalength = mex32.inputdatalength; - mex64.outputdata = compat_ptr(mex32.outputdata); - mex64.outputdatalength = mex32.outputdatalength; - mex64.b_key = compat_ptr(mex32.b_key); - mex64.n_modulus = compat_ptr(mex32.n_modulus); - do { - rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) - do { - rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - if (rc) - return rc; - return put_user(mex64.outputdatalength, - &umex32->outputdatalength); -} - -struct compat_ica_rsa_modexpo_crt { - compat_uptr_t inputdata; - unsigned int inputdatalength; - compat_uptr_t outputdata; - unsigned int outputdatalength; - compat_uptr_t bp_key; - compat_uptr_t bq_key; - compat_uptr_t np_prime; - compat_uptr_t nq_prime; - compat_uptr_t u_mult_inv; -}; - -static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_rsa_modexpo_crt __user *ucrt32 = compat_ptr(arg); - struct compat_ica_rsa_modexpo_crt crt32; - struct ica_rsa_modexpo_crt crt64; - struct zcrypt_track tr; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&crt32, ucrt32, sizeof(crt32))) - return -EFAULT; - crt64.inputdata = compat_ptr(crt32.inputdata); - crt64.inputdatalength = crt32.inputdatalength; - crt64.outputdata = compat_ptr(crt32.outputdata); - crt64.outputdatalength = crt32.outputdatalength; - crt64.bp_key = compat_ptr(crt32.bp_key); - crt64.bq_key = compat_ptr(crt32.bq_key); - crt64.np_prime = compat_ptr(crt32.np_prime); - crt64.nq_prime = compat_ptr(crt32.nq_prime); - crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv); - do { - rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) - do { - rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - if (rc) - return rc; - return put_user(crt64.outputdatalength, - &ucrt32->outputdatalength); -} - -struct compat_ica_xcrb { - unsigned short agent_ID; - unsigned int user_defined; - unsigned short request_ID; - unsigned int request_control_blk_length; - unsigned char padding1[16 - sizeof(compat_uptr_t)]; - compat_uptr_t request_control_blk_addr; - unsigned int request_data_length; - char padding2[16 - sizeof(compat_uptr_t)]; - compat_uptr_t request_data_address; - unsigned int reply_control_blk_length; - char padding3[16 - sizeof(compat_uptr_t)]; - compat_uptr_t reply_control_blk_addr; - unsigned int reply_data_length; - char padding4[16 - sizeof(compat_uptr_t)]; - compat_uptr_t reply_data_addr; - unsigned short priority_window; - unsigned int status; -} __packed; - -static long trans_xcrb32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_xcrb __user *uxcrb32 = compat_ptr(arg); - struct compat_ica_xcrb xcrb32; - struct zcrypt_track tr; - struct ica_xcRB xcrb64; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&xcrb32, uxcrb32, sizeof(xcrb32))) - return -EFAULT; - xcrb64.agent_ID = xcrb32.agent_ID; - xcrb64.user_defined = xcrb32.user_defined; - xcrb64.request_ID = xcrb32.request_ID; - xcrb64.request_control_blk_length = - xcrb32.request_control_blk_length; - xcrb64.request_control_blk_addr = - compat_ptr(xcrb32.request_control_blk_addr); - xcrb64.request_data_length = - xcrb32.request_data_length; - xcrb64.request_data_address = - compat_ptr(xcrb32.request_data_address); - xcrb64.reply_control_blk_length = - xcrb32.reply_control_blk_length; - xcrb64.reply_control_blk_addr = - compat_ptr(xcrb32.reply_control_blk_addr); - xcrb64.reply_data_length = xcrb32.reply_data_length; - xcrb64.reply_data_addr = - compat_ptr(xcrb32.reply_data_addr); - xcrb64.priority_window = xcrb32.priority_window; - xcrb64.status = xcrb32.status; - do { - rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) - do { - rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - xcrb32.reply_control_blk_length = xcrb64.reply_control_blk_length; - xcrb32.reply_data_length = xcrb64.reply_data_length; - xcrb32.status = xcrb64.status; - if (copy_to_user(uxcrb32, &xcrb32, sizeof(xcrb32))) - return -EFAULT; - return rc; -} - -static long zcrypt_compat_ioctl(struct file *filp, unsigned int cmd, - unsigned long arg) -{ - int rc; - struct ap_perms *perms = - (struct ap_perms *)filp->private_data; - - rc = zcrypt_check_ioctl(perms, cmd); - if (rc) - return rc; - - if (cmd == ICARSAMODEXPO) - return trans_modexpo32(perms, filp, cmd, arg); - if (cmd == ICARSACRT) - return trans_modexpo_crt32(perms, filp, cmd, arg); - if (cmd == ZSECSENDCPRB) - return trans_xcrb32(perms, filp, cmd, arg); - return zcrypt_unlocked_ioctl(filp, cmd, arg); -} -#endif - /* * Misc device file operations. */ @@ -1885,12 +1740,8 @@ static const struct file_operations zcrypt_fops = { .read = zcrypt_read, .write = zcrypt_write, .unlocked_ioctl = zcrypt_unlocked_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = zcrypt_compat_ioctl, -#endif .open = zcrypt_open, .release = zcrypt_release, - .llseek = no_llseek, }; /* @@ -1917,8 +1768,8 @@ static int zcrypt_rng_data_read(struct hwrng *rng, u32 *data) */ if (zcrypt_rng_buffer_index == 0) { rc = zcrypt_rng((char *)zcrypt_rng_buffer); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + /* on ENODEV failure: retry once again after an AP bus rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) rc = zcrypt_rng((char *)zcrypt_rng_buffer); if (rc < 0) return -EIO; @@ -1980,7 +1831,7 @@ void zcrypt_rng_device_remove(void) * an asynchronous job. This function waits until these initial jobs * are done and so the zcrypt api should be ready to serve crypto * requests - if there are resources available. The function uses an - * internal timeout of 60s. The very first caller will either wait for + * internal timeout of 30s. The very first caller will either wait for * ap bus bindings complete or the timeout happens. This state will be * remembered for further callers which will only be blocked until a * decision is made (timeout or bindings complete). @@ -1999,8 +1850,8 @@ int zcrypt_wait_api_operational(void) switch (zcrypt_wait_api_state) { case 0: /* initial state, invoke wait for the ap bus complete */ - rc = ap_wait_init_apqn_bindings_complete( - msecs_to_jiffies(60 * 1000)); + rc = ap_wait_apqn_bindings_complete( + msecs_to_jiffies(ZCRYPT_WAIT_BINDINGS_COMPLETE_MS)); switch (rc) { case 0: /* ap bus bindings are complete */ @@ -2017,8 +1868,7 @@ int zcrypt_wait_api_operational(void) break; default: /* other failure */ - ZCRYPT_DBF_DBG("%s ap_wait_init_apqn_bindings_complete()=%d\n", - __func__, rc); + pr_debug("ap_wait_init_apqn_bindings_complete()=%d\n", rc); break; } break; @@ -2041,7 +1891,7 @@ EXPORT_SYMBOL(zcrypt_wait_api_operational); int __init zcrypt_debug_init(void) { zcrypt_dbf_info = debug_register("zcrypt", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + ZCRYPT_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(zcrypt_dbf_info, &debug_sprintf_view); debug_set_level(zcrypt_dbf_info, DBF_ERR); @@ -2058,12 +1908,9 @@ static int __init zcdn_init(void) int rc; /* create a new class 'zcrypt' */ - zcrypt_class = class_create(ZCRYPT_NAME); - if (IS_ERR(zcrypt_class)) { - rc = PTR_ERR(zcrypt_class); - goto out_class_create_failed; - } - zcrypt_class->dev_release = zcdn_device_release; + rc = class_register(&zcrypt_class); + if (rc) + goto out_class_register_failed; /* alloc device minor range */ rc = alloc_chrdev_region(&zcrypt_devt, @@ -2079,35 +1926,35 @@ static int __init zcdn_init(void) goto out_cdev_add_failed; /* need some class specific sysfs attributes */ - rc = class_create_file(zcrypt_class, &class_attr_zcdn_create); + rc = class_create_file(&zcrypt_class, &class_attr_zcdn_create); if (rc) goto out_class_create_file_1_failed; - rc = class_create_file(zcrypt_class, &class_attr_zcdn_destroy); + rc = class_create_file(&zcrypt_class, &class_attr_zcdn_destroy); if (rc) goto out_class_create_file_2_failed; return 0; out_class_create_file_2_failed: - class_remove_file(zcrypt_class, &class_attr_zcdn_create); + class_remove_file(&zcrypt_class, &class_attr_zcdn_create); out_class_create_file_1_failed: cdev_del(&zcrypt_cdev); out_cdev_add_failed: unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES); out_alloc_chrdev_failed: - class_destroy(zcrypt_class); -out_class_create_failed: + class_unregister(&zcrypt_class); +out_class_register_failed: return rc; } static void zcdn_exit(void) { - class_remove_file(zcrypt_class, &class_attr_zcdn_create); - class_remove_file(zcrypt_class, &class_attr_zcdn_destroy); + class_remove_file(&zcrypt_class, &class_attr_zcdn_create); + class_remove_file(&zcrypt_class, &class_attr_zcdn_destroy); zcdn_destroy_all(); cdev_del(&zcrypt_cdev); unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES); - class_destroy(zcrypt_class); + class_unregister(&zcrypt_class); } /* @@ -2119,13 +1966,27 @@ int __init zcrypt_api_init(void) { int rc; + /* make sure the mempool threshold is >= 1 */ + if (zcrypt_mempool_threshold < 1) { + rc = -EINVAL; + goto out; + } + rc = zcrypt_debug_init(); if (rc) goto out; rc = zcdn_init(); if (rc) - goto out; + goto out_zcdn_init_failed; + + rc = zcrypt_ccamisc_init(); + if (rc) + goto out_ccamisc_init_failed; + + rc = zcrypt_ep11misc_init(); + if (rc) + goto out_ep11misc_init_failed; /* Register the request sprayer. */ rc = misc_register(&zcrypt_misc_device); @@ -2138,7 +1999,12 @@ int __init zcrypt_api_init(void) return 0; out_misc_register_failed: + zcrypt_ep11misc_exit(); +out_ep11misc_init_failed: + zcrypt_ccamisc_exit(); +out_ccamisc_init_failed: zcdn_exit(); +out_zcdn_init_failed: zcrypt_debug_exit(); out: return rc; diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index de659954c8f7..6ef8850a42df 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -38,6 +38,15 @@ */ #define ZCRYPT_RNG_BUFFER_SIZE 4096 +/** + * The zcrypt_wait_api_operational() function waits this + * amount in milliseconds for ap_wait_aqpn_bindings_complete(). + * Also on a cprb send failure with ENODEV the send functions + * trigger an ap bus rescan and wait this time in milliseconds + * for ap_wait_aqpn_bindings_complete() before resending. + */ +#define ZCRYPT_WAIT_BINDINGS_COMPLETE_MS 30000 + /* * Identifier for Crypto Request Performance Index */ @@ -67,6 +76,13 @@ struct zcrypt_track { #define TRACK_AGAIN_CARD_WEIGHT_PENALTY 1000 #define TRACK_AGAIN_QUEUE_WEIGHT_PENALTY 10000 +/* + * xflags - to be used with zcrypt_send_cprb() and + * zcrypt_send_ep11_cprb() for the xflags parameter. + */ +#define ZCRYPT_XFLAG_USERSPACE 0x0001 /* data ptrs address userspace */ +#define ZCRYPT_XFLAG_NOMEMALLOC 0x0002 /* do not allocate memory via kmalloc */ + struct zcrypt_ops { long (*rsa_modexpo)(struct zcrypt_queue *, struct ica_rsa_modexpo *, struct ap_message *); @@ -123,6 +139,8 @@ extern atomic_t zcrypt_rescan_req; extern spinlock_t zcrypt_list_lock; extern struct list_head zcrypt_card_list; +extern unsigned int zcrypt_mempool_threshold; + #define for_each_zcrypt_card(_zc) \ list_for_each_entry(_zc, &zcrypt_card_list, list) @@ -152,9 +170,10 @@ void zcrypt_msgtype_unregister(struct zcrypt_ops *); struct zcrypt_ops *zcrypt_msgtype(unsigned char *, int); int zcrypt_api_init(void); void zcrypt_api_exit(void); -long zcrypt_send_cprb(struct ica_xcRB *xcRB); -long zcrypt_send_ep11_cprb(struct ep11_urb *urb); -void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus); +long zcrypt_send_cprb(struct ica_xcRB *xcRB, u32 xflags); +long zcrypt_send_ep11_cprb(struct ep11_urb *urb, u32 xflags); +void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus, + int maxcard, int maxqueue); int zcrypt_device_status_ext(int card, int queue, struct zcrypt_device_status_ext *devstatus); diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index c815722d0ac8..6dea702a5cac 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -11,6 +11,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ +#include <linux/export.h> #include <linux/module.h> #include <linux/init.h> #include <linux/interrupt.h> @@ -18,7 +19,6 @@ #include <linux/fs.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> @@ -52,7 +52,7 @@ static ssize_t online_show(struct device *dev, { struct zcrypt_card *zc = dev_get_drvdata(dev); struct ap_card *ac = to_ap_card(dev); - int online = ac->config && zc->online ? 1 : 0; + int online = ac->config && !ac->chkstop && zc->online ? 1 : 0; return sysfs_emit(buf, "%d\n", online); } @@ -70,7 +70,7 @@ static ssize_t online_store(struct device *dev, if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1) return -EINVAL; - if (online && !ac->config) + if (online && (!ac->config || ac->chkstop)) return -ENODEV; zc->online = online; diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 263fe182648b..573bad1d6d86 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -7,10 +7,11 @@ * Collection of CCA misc functions used by zcrypt and pkey */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt +#include <linux/export.h> #include <linux/init.h> +#include <linux/mempool.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/random.h> @@ -23,27 +24,37 @@ #include "zcrypt_msgtype6.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - /* Size of parameter block used for all cca requests/replies */ #define PARMBSIZE 512 /* Size of vardata block used for some of the cca requests/replies */ #define VARDATASIZE 4096 -struct cca_info_list_entry { - struct list_head list; - u16 cardnr; - u16 domain; - struct cca_info info; -}; +/* + * Cprb memory pool held for urgent cases where no memory + * can be allocated via kmalloc. This pool is only used + * when alloc_and_prep_cprbmem() is called with the xflag + * ZCRYPT_XFLAG_NOMEMALLOC. The cprb memory needs to hold + * space for request AND reply! + */ +#define CPRB_MEMPOOL_ITEM_SIZE (16 * 1024) +static mempool_t *cprb_mempool; -/* a list with cca_info_list_entry entries */ -static LIST_HEAD(cca_info_list); -static DEFINE_SPINLOCK(cca_info_list_lock); +/* + * This is a pre-allocated memory for the device status array + * used within the findcard() functions. It is currently + * 128 * 128 * 4 bytes = 64 KB big. Usage of this memory is + * controlled via dev_status_mem_mutex. Needs adaption if more + * than 128 cards or domains to be are supported. + */ +#define ZCRYPT_DEV_STATUS_CARD_MAX 128 +#define ZCRYPT_DEV_STATUS_QUEUE_MAX 128 +#define ZCRYPT_DEV_STATUS_ENTRIES (ZCRYPT_DEV_STATUS_CARD_MAX * \ + ZCRYPT_DEV_STATUS_QUEUE_MAX) +#define ZCRYPT_DEV_STATUS_EXT_SIZE (ZCRYPT_DEV_STATUS_ENTRIES * \ + sizeof(struct zcrypt_device_status_ext)) +static void *dev_status_mem; +static DEFINE_MUTEX(dev_status_mem_mutex); /* * Simple check if the token is a valid CCA secure AES data key @@ -177,7 +188,7 @@ EXPORT_SYMBOL(cca_check_secaescipherkey); * key token. Returns 0 on success or errno value on failure. */ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, size_t keysize, + const u8 *token, u32 keysize, int checkcpacfexport) { struct eccprivkeytoken *t = (struct eccprivkeytoken *)token; @@ -192,7 +203,7 @@ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, } if (t->len > keysize) { if (dbg) - DBF("%s token check failed, len %d > keysize %zu\n", + DBF("%s token check failed, len %d > keysize %u\n", __func__, (int)t->len, keysize); return -EINVAL; } @@ -224,19 +235,27 @@ EXPORT_SYMBOL(cca_check_sececckeytoken); static int alloc_and_prep_cprbmem(size_t paramblen, u8 **p_cprb_mem, struct CPRBX **p_req_cprb, - struct CPRBX **p_rep_cprb) + struct CPRBX **p_rep_cprb, + u32 xflags) { - u8 *cprbmem; + u8 *cprbmem = NULL; size_t cprbplusparamblen = sizeof(struct CPRBX) + paramblen; + size_t len = 2 * cprbplusparamblen; struct CPRBX *preqcblk, *prepcblk; /* * allocate consecutive memory for request CPRB, request param * block, reply CPRB and reply param block */ - cprbmem = kcalloc(2, cprbplusparamblen, GFP_KERNEL); + if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) { + if (len <= CPRB_MEMPOOL_ITEM_SIZE) + cprbmem = mempool_alloc_preallocated(cprb_mempool); + } else { + cprbmem = kmalloc(len, GFP_KERNEL); + } if (!cprbmem) return -ENOMEM; + memset(cprbmem, 0, len); preqcblk = (struct CPRBX *)cprbmem; prepcblk = (struct CPRBX *)(cprbmem + cprbplusparamblen); @@ -266,11 +285,15 @@ static int alloc_and_prep_cprbmem(size_t paramblen, * with zeros before freeing (useful if there was some * clear key material in there). */ -static void free_cprbmem(void *mem, size_t paramblen, int scrub) +static void free_cprbmem(void *mem, size_t paramblen, bool scrub, u32 xflags) { - if (scrub) + if (mem && scrub) memzero_explicit(mem, 2 * (sizeof(struct CPRBX) + paramblen)); - kfree(mem); + + if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) + mempool_free(mem, cprb_mempool); + else + kfree(mem); } /* @@ -295,7 +318,7 @@ static inline void prep_xcrb(struct ica_xcRB *pxcrb, * Generate (random) CCA AES DATA secure key. */ int cca_genseckey(u16 cardnr, u16 domain, - u32 keybitsize, u8 *seckey) + u32 keybitsize, u8 *seckey, u32 xflags) { int i, rc, keysize; int seckeysize; @@ -337,7 +360,8 @@ int cca_genseckey(u16 cardnr, u16 domain, } __packed * prepparm; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -367,8 +391,8 @@ int cca_genseckey(u16 cardnr, u16 domain, memcpy(preqparm->lv1.key_length, "KEYLN32 ", 8); break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -384,17 +408,17 @@ int cca_genseckey(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s secure key generate failure, card response %d/%d\n", - __func__, + ZCRYPT_DBF_ERR("%s secure key generate failure, card response %d/%d\n", + __func__, (int)prepcblk->ccp_rtcode, (int)prepcblk->ccp_rscode); rc = -EIO; @@ -411,8 +435,8 @@ int cca_genseckey(u16 cardnr, u16 domain, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -429,7 +453,7 @@ int cca_genseckey(u16 cardnr, u16 domain, memcpy(seckey, prepparm->lv3.keyblock.tok, SECKEYBLOBSIZE); out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, false, xflags); return rc; } EXPORT_SYMBOL(cca_genseckey); @@ -438,7 +462,7 @@ EXPORT_SYMBOL(cca_genseckey); * Generate an CCA AES DATA secure key with given key value. */ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, - const u8 *clrkey, u8 *seckey) + const u8 *clrkey, u8 *seckey, u32 xflags) { int rc, keysize, seckeysize; u8 *mem, *ptr; @@ -478,7 +502,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, } __packed * prepparm; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -505,8 +530,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, keysize = 32; break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -522,19 +547,19 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s clear key import failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s clear key import failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -549,8 +574,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -568,7 +593,7 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, memcpy(seckey, prepparm->lv3.keyblock.tok, SECKEYBLOBSIZE); out: - free_cprbmem(mem, PARMBSIZE, 1); + free_cprbmem(mem, PARMBSIZE, true, xflags); return rc; } EXPORT_SYMBOL(cca_clr2seckey); @@ -578,7 +603,7 @@ EXPORT_SYMBOL(cca_clr2seckey); */ int cca_sec2protkey(u16 cardnr, u16 domain, const u8 *seckey, u8 *protkey, u32 *protkeylen, - u32 *protkeytype) + u32 *protkeytype, u32 xflags) { int rc; u8 *mem, *ptr; @@ -624,7 +649,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, } __packed * prepparm; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -649,30 +675,30 @@ int cca_sec2protkey(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) - rc = -EAGAIN; + rc = -EBUSY; else rc = -EIO; goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN("%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -683,8 +709,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, /* check the returned keyblock */ if (prepparm->lv3.ckb.version != 0x01 && prepparm->lv3.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->lv3.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->lv3.ckb.version); rc = -EIO; goto out; } @@ -707,8 +733,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->lv3.ckb.len); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->lv3.ckb.len); rc = -EIO; goto out; } @@ -717,7 +743,7 @@ int cca_sec2protkey(u16 cardnr, u16 domain, *protkeylen = prepparm->lv3.ckb.len; out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, true, xflags); return rc; } EXPORT_SYMBOL(cca_sec2protkey); @@ -742,7 +768,7 @@ static const u8 aes_cipher_key_skeleton[] = { * Generate (random) CCA AES CIPHER secure key. */ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize) + u8 *keybuf, u32 *keybufsize, u32 xflags) { int rc; u8 *mem, *ptr; @@ -818,7 +844,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, struct cipherkeytoken *t; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -840,9 +867,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -878,21 +904,19 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s cipher key generate failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s cipher key generate failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -905,8 +929,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 5 * sizeof(uint16_t) || prepparm->kb.len > 136 + 5 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -931,7 +955,7 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, *keybufsize = t->len; out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, false, xflags); return rc; } EXPORT_SYMBOL(cca_gencipherkey); @@ -946,7 +970,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, const u8 *clr_key_value, int clr_key_bit_size, u8 *key_token, - int *key_token_size) + int *key_token_size, + u32 xflags) { int rc, n; u8 *mem, *ptr; @@ -997,7 +1022,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, int complete = strncmp(rule_array_2, "COMPLETE", 8) ? 0 : 1; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -1046,21 +1072,19 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s CSNBKPI2 failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s CSNBKPI2 failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -1073,8 +1097,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 3 * sizeof(uint16_t) || prepparm->kb.len > 136 + 3 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -1087,7 +1111,7 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, *key_token_size = t->len; out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, false, xflags); return rc; } @@ -1095,23 +1119,31 @@ out: * Build CCA AES CIPHER secure key with a given clear key value. */ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize) + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, u32 xflags) { int rc; - u8 *token; + void *mem; int tokensize; - u8 exorbuf[32]; + u8 *token, exorbuf[32]; struct cipherkeytoken *t; /* fill exorbuf with random data */ get_random_bytes(exorbuf, sizeof(exorbuf)); - /* allocate space for the key token to build */ - token = kmalloc(MAXCCAVLSCTOKENSIZE, GFP_KERNEL); - if (!token) + /* + * Allocate space for the key token to build. + * Also we only need up to MAXCCAVLSCTOKENSIZE bytes for this + * we use the already existing cprb mempool to solve this + * short term memory requirement. + */ + mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ? + mempool_alloc_preallocated(cprb_mempool) : + mempool_alloc(cprb_mempool, GFP_KERNEL); + if (!mem) return -ENOMEM; /* prepare the token with the key skeleton */ + token = (u8 *)mem; tokensize = SIZEOF_SKELETON; memcpy(token, aes_cipher_key_skeleton, tokensize); @@ -1130,35 +1162,31 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, * 4/4 COMPLETE the secure cipher key import */ rc = _ip_cprb_helper(card, dom, "AES ", "FIRST ", "MIN3PART", - exorbuf, keybitsize, token, &tokensize); + exorbuf, keybitsize, token, &tokensize, xflags); if (rc) { - DEBUG_ERR( - "%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, - clrkey, keybitsize, token, &tokensize); + clrkey, keybitsize, token, &tokensize, xflags); if (rc) { - DEBUG_ERR( - "%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, - exorbuf, keybitsize, token, &tokensize); + exorbuf, keybitsize, token, &tokensize, xflags); if (rc) { - DEBUG_ERR( - "%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "COMPLETE", NULL, - NULL, keybitsize, token, &tokensize); + NULL, keybitsize, token, &tokensize, xflags); if (rc) { - DEBUG_ERR( - "%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } @@ -1172,7 +1200,7 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, *keybufsize = tokensize; out: - kfree(token); + mempool_free(mem, cprb_mempool); return rc; } EXPORT_SYMBOL(cca_clr2cipherkey); @@ -1181,7 +1209,8 @@ EXPORT_SYMBOL(cca_clr2cipherkey); * Derive proteced key from CCA AES cipher secure key. */ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags) { int rc; u8 *mem, *ptr; @@ -1233,7 +1262,8 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, int keytoklen = ((struct cipherkeytoken *)ckey)->len; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -1263,33 +1293,30 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) - rc = -EAGAIN; + rc = -EBUSY; else rc = -EIO; goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1300,15 +1327,14 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x01 && prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x02) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } @@ -1331,8 +1357,8 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->vud.ckb.keylen); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->vud.ckb.keylen); rc = -EIO; goto out; } @@ -1341,7 +1367,7 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, *protkeylen = prepparm->vud.ckb.keylen; out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, true, xflags); return rc; } EXPORT_SYMBOL(cca_cipher2protkey); @@ -1350,7 +1376,7 @@ EXPORT_SYMBOL(cca_cipher2protkey); * Derive protected key from CCA ECC secure private key. */ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) + u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags) { int rc; u8 *mem, *ptr; @@ -1400,7 +1426,8 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, int keylen = ((struct eccprivkeytoken *)key)->len; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -1430,33 +1457,30 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) - rc = -EAGAIN; + rc = -EBUSY; else rc = -EIO; goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1466,23 +1490,22 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x81) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x81\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x81\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } /* copy the translated protected key */ if (prepparm->vud.ckb.keylen > *protkeylen) { - DEBUG_ERR("%s prot keylen mismatch %d > buffersize %u\n", - __func__, prepparm->vud.ckb.keylen, *protkeylen); + ZCRYPT_DBF_ERR("%s prot keylen mismatch %d > buffersize %u\n", + __func__, prepparm->vud.ckb.keylen, *protkeylen); rc = -EIO; goto out; } @@ -1492,7 +1515,7 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, *protkeytype = PKEY_KEYTYPE_ECC; out: - free_cprbmem(mem, PARMBSIZE, 0); + free_cprbmem(mem, PARMBSIZE, true, xflags); return rc; } EXPORT_SYMBOL(cca_ecc2protkey); @@ -1503,7 +1526,8 @@ EXPORT_SYMBOL(cca_ecc2protkey); int cca_query_crypto_facility(u16 cardnr, u16 domain, const char *keyword, u8 *rarray, size_t *rarraylen, - u8 *varray, size_t *varraylen) + u8 *varray, size_t *varraylen, + u32 xflags) { int rc; u16 len; @@ -1527,7 +1551,8 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, } __packed * prepparm; /* get already prepared memory for 2 cprbs with param block each */ - rc = alloc_and_prep_cprbmem(parmbsize, &mem, &preqcblk, &prepcblk); + rc = alloc_and_prep_cprbmem(parmbsize, &mem, + &preqcblk, &prepcblk, xflags); if (rc) return rc; @@ -1548,19 +1573,19 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb, xflags); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -1595,94 +1620,21 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, } out: - free_cprbmem(mem, parmbsize, 0); + free_cprbmem(mem, parmbsize, false, xflags); return rc; } EXPORT_SYMBOL(cca_query_crypto_facility); -static int cca_info_cache_fetch(u16 cardnr, u16 domain, struct cca_info *ci) -{ - int rc = -ENOENT; - struct cca_info_list_entry *ptr; - - spin_lock_bh(&cca_info_list_lock); - list_for_each_entry(ptr, &cca_info_list, list) { - if (ptr->cardnr == cardnr && ptr->domain == domain) { - memcpy(ci, &ptr->info, sizeof(*ci)); - rc = 0; - break; - } - } - spin_unlock_bh(&cca_info_list_lock); - - return rc; -} - -static void cca_info_cache_update(u16 cardnr, u16 domain, - const struct cca_info *ci) -{ - int found = 0; - struct cca_info_list_entry *ptr; - - spin_lock_bh(&cca_info_list_lock); - list_for_each_entry(ptr, &cca_info_list, list) { - if (ptr->cardnr == cardnr && - ptr->domain == domain) { - memcpy(&ptr->info, ci, sizeof(*ci)); - found = 1; - break; - } - } - if (!found) { - ptr = kmalloc(sizeof(*ptr), GFP_ATOMIC); - if (!ptr) { - spin_unlock_bh(&cca_info_list_lock); - return; - } - ptr->cardnr = cardnr; - ptr->domain = domain; - memcpy(&ptr->info, ci, sizeof(*ci)); - list_add(&ptr->list, &cca_info_list); - } - spin_unlock_bh(&cca_info_list_lock); -} - -static void cca_info_cache_scrub(u16 cardnr, u16 domain) -{ - struct cca_info_list_entry *ptr; - - spin_lock_bh(&cca_info_list_lock); - list_for_each_entry(ptr, &cca_info_list, list) { - if (ptr->cardnr == cardnr && - ptr->domain == domain) { - list_del(&ptr->list); - kfree(ptr); - break; - } - } - spin_unlock_bh(&cca_info_list_lock); -} - -static void __exit mkvp_cache_free(void) -{ - struct cca_info_list_entry *ptr, *pnext; - - spin_lock_bh(&cca_info_list_lock); - list_for_each_entry_safe(ptr, pnext, &cca_info_list, list) { - list_del(&ptr->list); - kfree(ptr); - } - spin_unlock_bh(&cca_info_list_lock); -} - /* - * Fetch cca_info values via query_crypto_facility from adapter. + * Fetch cca_info values about a CCA queue via + * query_crypto_facility from adapter. */ -static int fetch_cca_info(u16 cardnr, u16 domain, struct cca_info *ci) +int cca_get_info(u16 cardnr, u16 domain, struct cca_info *ci, u32 xflags) { + void *mem; int rc, found = 0; size_t rlen, vlen; - u8 *rarray, *varray, *pg; + u8 *rarray, *varray; struct zcrypt_device_status_ext devstat; memset(ci, 0, sizeof(*ci)); @@ -1693,17 +1645,22 @@ static int fetch_cca_info(u16 cardnr, u16 domain, struct cca_info *ci) return rc; ci->hwtype = devstat.hwtype; - /* prep page for rule array and var array use */ - pg = (u8 *)__get_free_page(GFP_KERNEL); - if (!pg) + /* + * Prep memory for rule array and var array use. + * Use the cprb mempool for this. + */ + mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ? + mempool_alloc_preallocated(cprb_mempool) : + mempool_alloc(cprb_mempool, GFP_KERNEL); + if (!mem) return -ENOMEM; - rarray = pg; - varray = pg + PAGE_SIZE / 2; + rarray = (u8 *)mem; + varray = (u8 *)mem + PAGE_SIZE / 2; rlen = vlen = PAGE_SIZE / 2; /* QF for this card/domain */ rc = cca_query_crypto_facility(cardnr, domain, "STATICSA", - rarray, &rlen, varray, &vlen); + rarray, &rlen, varray, &vlen, xflags); if (rc == 0 && rlen >= 10 * 8 && vlen >= 204) { memcpy(ci->serial, rarray, 8); ci->new_asym_mk_state = (char)rarray[4 * 8]; @@ -1730,7 +1687,7 @@ static int fetch_cca_info(u16 cardnr, u16 domain, struct cca_info *ci) goto out; rlen = vlen = PAGE_SIZE / 2; rc = cca_query_crypto_facility(cardnr, domain, "STATICSB", - rarray, &rlen, varray, &vlen); + rarray, &rlen, varray, &vlen, xflags); if (rc == 0 && rlen >= 13 * 8 && vlen >= 240) { ci->new_apka_mk_state = (char)rarray[10 * 8]; ci->cur_apka_mk_state = (char)rarray[11 * 8]; @@ -1745,177 +1702,32 @@ static int fetch_cca_info(u16 cardnr, u16 domain, struct cca_info *ci) } out: - free_page((unsigned long)pg); + mempool_free(mem, cprb_mempool); return found == 2 ? 0 : -ENOENT; } - -/* - * Fetch cca information about a CCA queue. - */ -int cca_get_info(u16 card, u16 dom, struct cca_info *ci, int verify) -{ - int rc; - - rc = cca_info_cache_fetch(card, dom, ci); - if (rc || verify) { - rc = fetch_cca_info(card, dom, ci); - if (rc == 0) - cca_info_cache_update(card, dom, ci); - } - - return rc; -} EXPORT_SYMBOL(cca_get_info); -/* - * Search for a matching crypto card based on the - * Master Key Verification Pattern given. - */ -static int findcard(u64 mkvp, u16 *pcardnr, u16 *pdomain, - int verify, int minhwtype) -{ - struct zcrypt_device_status_ext *device_status; - u16 card, dom; - struct cca_info ci; - int i, rc, oi = -1; - - /* mkvp must not be zero, minhwtype needs to be >= 0 */ - if (mkvp == 0 || minhwtype < 0) - return -EINVAL; - - /* fetch status of all crypto cards */ - device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT, - sizeof(struct zcrypt_device_status_ext), - GFP_KERNEL); - if (!device_status) - return -ENOMEM; - zcrypt_device_status_mask_ext(device_status); - - /* walk through all crypto cards */ - for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { - card = AP_QID_CARD(device_status[i].qid); - dom = AP_QID_QUEUE(device_status[i].qid); - if (device_status[i].online && - device_status[i].functions & 0x04) { - /* enabled CCA card, check current mkvp from cache */ - if (cca_info_cache_fetch(card, dom, &ci) == 0 && - ci.hwtype >= minhwtype && - ci.cur_aes_mk_state == '2' && - ci.cur_aes_mkvp == mkvp) { - if (!verify) - break; - /* verify: refresh card info */ - if (fetch_cca_info(card, dom, &ci) == 0) { - cca_info_cache_update(card, dom, &ci); - if (ci.hwtype >= minhwtype && - ci.cur_aes_mk_state == '2' && - ci.cur_aes_mkvp == mkvp) - break; - } - } - } else { - /* Card is offline and/or not a CCA card. */ - /* del mkvp entry from cache if it exists */ - cca_info_cache_scrub(card, dom); - } - } - if (i >= MAX_ZDEV_ENTRIES_EXT) { - /* nothing found, so this time without cache */ - for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { - if (!(device_status[i].online && - device_status[i].functions & 0x04)) - continue; - card = AP_QID_CARD(device_status[i].qid); - dom = AP_QID_QUEUE(device_status[i].qid); - /* fresh fetch mkvp from adapter */ - if (fetch_cca_info(card, dom, &ci) == 0) { - cca_info_cache_update(card, dom, &ci); - if (ci.hwtype >= minhwtype && - ci.cur_aes_mk_state == '2' && - ci.cur_aes_mkvp == mkvp) - break; - if (ci.hwtype >= minhwtype && - ci.old_aes_mk_state == '2' && - ci.old_aes_mkvp == mkvp && - oi < 0) - oi = i; - } - } - if (i >= MAX_ZDEV_ENTRIES_EXT && oi >= 0) { - /* old mkvp matched, use this card then */ - card = AP_QID_CARD(device_status[oi].qid); - dom = AP_QID_QUEUE(device_status[oi].qid); - } - } - if (i < MAX_ZDEV_ENTRIES_EXT || oi >= 0) { - if (pcardnr) - *pcardnr = card; - if (pdomain) - *pdomain = dom; - rc = (i < MAX_ZDEV_ENTRIES_EXT ? 0 : 1); - } else { - rc = -ENODEV; - } - - kvfree(device_status); - return rc; -} - -/* - * Search for a matching crypto card based on the Master Key - * Verification Pattern provided inside a secure key token. - */ -int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify) -{ - u64 mkvp; - int minhwtype = 0; - const struct keytoken_header *hdr = (struct keytoken_header *)key; - - if (hdr->type != TOKTYPE_CCA_INTERNAL) - return -EINVAL; - - switch (hdr->version) { - case TOKVER_CCA_AES: - mkvp = ((struct secaeskeytoken *)key)->mkvp; - break; - case TOKVER_CCA_VLSC: - mkvp = ((struct cipherkeytoken *)key)->mkvp0; - minhwtype = AP_DEVICE_TYPE_CEX6; - break; - default: - return -EINVAL; - } - - return findcard(mkvp, pcardnr, pdomain, verify, minhwtype); -} -EXPORT_SYMBOL(cca_findcard); - -int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, +int cca_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain, int minhwtype, int mktype, u64 cur_mkvp, u64 old_mkvp, - int verify) + u32 xflags) { struct zcrypt_device_status_ext *device_status; - u32 *_apqns = NULL, _nr_apqns = 0; - int i, card, dom, curmatch, oldmatch, rc = 0; + int i, card, dom, curmatch, oldmatch; struct cca_info ci; + u32 _nr_apqns = 0; - /* fetch status of all crypto cards */ - device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT, - sizeof(struct zcrypt_device_status_ext), - GFP_KERNEL); - if (!device_status) - return -ENOMEM; - zcrypt_device_status_mask_ext(device_status); + /* occupy the device status memory */ + mutex_lock(&dev_status_mem_mutex); + memset(dev_status_mem, 0, ZCRYPT_DEV_STATUS_EXT_SIZE); + device_status = (struct zcrypt_device_status_ext *)dev_status_mem; - /* allocate 1k space for up to 256 apqns */ - _apqns = kmalloc_array(256, sizeof(u32), GFP_KERNEL); - if (!_apqns) { - kvfree(device_status); - return -ENOMEM; - } + /* fetch crypto device status into this struct */ + zcrypt_device_status_mask_ext(device_status, + ZCRYPT_DEV_STATUS_CARD_MAX, + ZCRYPT_DEV_STATUS_QUEUE_MAX); /* walk through all the crypto apqnss */ - for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { + for (i = 0; i < ZCRYPT_DEV_STATUS_ENTRIES; i++) { card = AP_QID_CARD(device_status[i].qid); dom = AP_QID_QUEUE(device_status[i].qid); /* check online state */ @@ -1931,7 +1743,7 @@ int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, if (domain != 0xFFFF && dom != domain) continue; /* get cca info on this apqn */ - if (cca_get_info(card, dom, &ci, verify)) + if (cca_get_info(card, dom, &ci, xflags)) continue; /* current master key needs to be valid */ if (mktype == AES_MK_SET && ci.cur_aes_mk_state != '2') @@ -1961,27 +1773,41 @@ int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, continue; } /* apqn passed all filtering criterons, add to the array */ - if (_nr_apqns < 256) - _apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom); + if (_nr_apqns < *nr_apqns) + apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom); } - /* nothing found ? */ - if (!_nr_apqns) { - kfree(_apqns); - rc = -ENODEV; - } else { - /* no re-allocation, simple return the _apqns array */ - *apqns = _apqns; - *nr_apqns = _nr_apqns; - rc = 0; - } + *nr_apqns = _nr_apqns; - kvfree(device_status); - return rc; + /* release the device status memory */ + mutex_unlock(&dev_status_mem_mutex); + + return _nr_apqns ? 0 : -ENODEV; } EXPORT_SYMBOL(cca_findcard2); -void __exit zcrypt_ccamisc_exit(void) +int __init zcrypt_ccamisc_init(void) +{ + /* Pre-allocate a small memory pool for cca cprbs. */ + cprb_mempool = mempool_create_kmalloc_pool(zcrypt_mempool_threshold, + CPRB_MEMPOOL_ITEM_SIZE); + if (!cprb_mempool) + return -ENOMEM; + + /* Pre-allocate one crypto status card struct used in findcard() */ + dev_status_mem = kvmalloc(ZCRYPT_DEV_STATUS_EXT_SIZE, GFP_KERNEL); + if (!dev_status_mem) { + mempool_destroy(cprb_mempool); + return -ENOMEM; + } + + return 0; +} + +void zcrypt_ccamisc_exit(void) { - mkvp_cache_free(); + mutex_lock(&dev_status_mem_mutex); + kvfree(dev_status_mem); + mutex_unlock(&dev_status_mem_mutex); + mempool_destroy(cprb_mempool); } diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 5ddf02f965f9..1ecc4e37e9ad 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -12,6 +12,7 @@ #include <asm/zcrypt.h> #include <asm/pkey.h> +#include "zcrypt_api.h" /* Key token types */ #define TOKTYPE_NON_CCA 0x00 /* Non-CCA key token */ @@ -153,50 +154,53 @@ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, * key token. Returns 0 on success or errno value on failure. */ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, size_t keysize, + const u8 *token, u32 keysize, int checkcpacfexport); /* * Generate (random) CCA AES DATA secure key. */ -int cca_genseckey(u16 cardnr, u16 domain, u32 keybitsize, u8 *seckey); +int cca_genseckey(u16 cardnr, u16 domain, u32 keybitsize, u8 *seckey, + u32 xflags); /* * Generate CCA AES DATA secure key with given clear key value. */ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, - const u8 *clrkey, u8 *seckey); + const u8 *clrkey, u8 *seckey, u32 xflags); /* * Derive proteced key from an CCA AES DATA secure key. */ int cca_sec2protkey(u16 cardnr, u16 domain, const u8 *seckey, u8 *protkey, u32 *protkeylen, - u32 *protkeytype); + u32 *protkeytype, u32 xflags); /* * Generate (random) CCA AES CIPHER secure key. */ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize); + u8 *keybuf, u32 *keybufsize, u32 xflags); /* * Derive proteced key from CCA AES cipher secure key. */ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags); /* * Build CCA AES CIPHER secure key with a given clear key value. */ int cca_clr2cipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize); + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, + u32 xflags); /* * Derive proteced key from CCA ECC secure private key. */ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); + u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags); /* * Query cryptographic facility from CCA adapter @@ -204,16 +208,8 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, int cca_query_crypto_facility(u16 cardnr, u16 domain, const char *keyword, u8 *rarray, size_t *rarraylen, - u8 *varray, size_t *varraylen); - -/* - * Search for a matching crypto card based on the Master Key - * Verification Pattern provided inside a secure key. - * Works with CCA AES data and cipher keys. - * Returns < 0 on failure, 0 if CURRENT MKVP matches and - * 1 if OLD MKVP matches. - */ -int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify); + u8 *varray, size_t *varraylen, + u32 xflags); /* * Build a list of cca apqns meeting the following constrains: @@ -223,21 +219,16 @@ int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify); * - if minhwtype > 0 only apqns with hwtype >= minhwtype * - if cur_mkvp != 0 only apqns where cur_mkvp == mkvp * - if old_mkvp != 0 only apqns where old_mkvp == mkvp - * - if verify is enabled and a cur_mkvp and/or old_mkvp - * value is given, then refetch the cca_info and make sure the current - * cur_mkvp or old_mkvp values of the apqn are used. * The mktype determines which set of master keys to use: * 0 = AES_MK_SET - AES MK set, 1 = APKA MK_SET - APKA MK set - * The array of apqn entries is allocated with kmalloc and returned in *apqns; - * the number of apqns stored into the list is returned in *nr_apqns. One apqn - * entry is simple a 32 bit value with 16 bit cardnr and 16 bit domain nr and - * may be casted to struct pkey_apqn. The return value is either 0 for success - * or a negative errno value. If no apqn meeting the criteria is found, - * -ENODEV is returned. + * The caller should set *nr_apqns to the nr of elements available in *apqns. + * On return *nr_apqns is then updated with the nr of apqns filled into *apqns. + * The return value is either 0 for success or a negative errno value. + * If no apqn meeting the criteria is found, -ENODEV is returned. */ -int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, +int cca_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain, int minhwtype, int mktype, u64 cur_mkvp, u64 old_mkvp, - int verify); + u32 xflags); #define AES_MK_SET 0 #define APKA_MK_SET 1 @@ -269,8 +260,9 @@ struct cca_info { /* * Fetch cca information about an CCA queue. */ -int cca_get_info(u16 card, u16 dom, struct cca_info *ci, int verify); +int cca_get_info(u16 card, u16 dom, struct cca_info *ci, u32 xflags); +int zcrypt_ccamisc_init(void); void zcrypt_ccamisc_exit(void); #endif /* _ZCRYPT_CCAMISC_H_ */ diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 9cfce9ff2e65..6ba7fbddd3f7 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -79,14 +79,13 @@ static ssize_t cca_serialnr_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = dev_get_drvdata(dev); - struct cca_info ci; struct ap_card *ac = to_ap_card(dev); + struct cca_info ci; memset(&ci, 0, sizeof(ci)); if (ap_domain_index >= 0) - cca_get_info(ac->id, ap_domain_index, &ci, zc->online); + cca_get_info(ac->id, ap_domain_index, &ci, 0); return sysfs_emit(buf, "%s\n", ci.serial); } @@ -110,17 +109,17 @@ static ssize_t cca_mkvps_show(struct device *dev, struct device_attribute *attr, char *buf) { + static const char * const new_state[] = { "empty", "partial", "full" }; + static const char * const cao_state[] = { "invalid", "valid" }; struct zcrypt_queue *zq = dev_get_drvdata(dev); - int n = 0; struct cca_info ci; - static const char * const cao_state[] = { "invalid", "valid" }; - static const char * const new_state[] = { "empty", "partial", "full" }; + int n = 0; memset(&ci, 0, sizeof(ci)); cca_get_info(AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), - &ci, zq->online); + &ci, 0); if (ci.new_aes_mk_state >= '1' && ci.new_aes_mk_state <= '3') n += sysfs_emit_at(buf, n, "AES NEW: %s 0x%016llx\n", @@ -210,13 +209,12 @@ static ssize_t ep11_api_ordinalnr_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = dev_get_drvdata(dev); - struct ep11_card_info ci; struct ap_card *ac = to_ap_card(dev); + struct ep11_card_info ci; memset(&ci, 0, sizeof(ci)); - ep11_get_card_info(ac->id, &ci, zc->online); + ep11_get_card_info(ac->id, &ci, 0); if (ci.API_ord_nr > 0) return sysfs_emit(buf, "%u\n", ci.API_ord_nr); @@ -231,13 +229,12 @@ static ssize_t ep11_fw_version_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = dev_get_drvdata(dev); - struct ep11_card_info ci; struct ap_card *ac = to_ap_card(dev); + struct ep11_card_info ci; memset(&ci, 0, sizeof(ci)); - ep11_get_card_info(ac->id, &ci, zc->online); + ep11_get_card_info(ac->id, &ci, 0); if (ci.FW_version > 0) return sysfs_emit(buf, "%d.%d\n", @@ -254,13 +251,12 @@ static ssize_t ep11_serialnr_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = dev_get_drvdata(dev); - struct ep11_card_info ci; struct ap_card *ac = to_ap_card(dev); + struct ep11_card_info ci; memset(&ci, 0, sizeof(ci)); - ep11_get_card_info(ac->id, &ci, zc->online); + ep11_get_card_info(ac->id, &ci, 0); if (ci.serial[0]) return sysfs_emit(buf, "%16.16s\n", ci.serial); @@ -279,7 +275,11 @@ static const struct { { 1, "BSI2009" }, { 2, "FIPS2011" }, { 3, "BSI2011" }, + { 4, "SIGG-IMPORT" }, + { 5, "SIGG" }, { 6, "BSICC2017" }, + { 7, "FIPS2021" }, + { 8, "FIPS2024" }, { 0, NULL } }; @@ -287,14 +287,13 @@ static ssize_t ep11_card_op_modes_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = dev_get_drvdata(dev); - int i, n = 0; - struct ep11_card_info ci; struct ap_card *ac = to_ap_card(dev); + struct ep11_card_info ci; + int i, n = 0; memset(&ci, 0, sizeof(ci)); - ep11_get_card_info(ac->id, &ci, zc->online); + ep11_get_card_info(ac->id, &ci, 0); for (i = 0; ep11_op_modes[i].mode_txt; i++) { if (ci.op_mode & (1ULL << ep11_op_modes[i].mode_bit)) { @@ -344,7 +343,7 @@ static ssize_t ep11_mkvps_show(struct device *dev, if (zq->online) ep11_get_domain_info(AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), - &di); + &di, 0); if (di.cur_wk_state == '0') { n = sysfs_emit(buf, "WK CUR: %s -\n", @@ -391,7 +390,7 @@ static ssize_t ep11_queue_op_modes_show(struct device *dev, if (zq->online) ep11_get_domain_info(AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), - &di); + &di, 0); for (i = 0; ep11_op_modes[i].mode_txt; i++) { if (di.op_mode & (1ULL << ep11_op_modes[i].mode_bit)) { @@ -473,7 +472,7 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) return -ENOMEM; zc->card = ac; dev_set_drvdata(&ap_dev->device, zc); - if (ap_test_bit(&ac->functions, AP_FUNC_ACCEL)) { + if (ac->hwinfo.accel) { if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) { zc->type_string = "CEX4A"; zc->user_space_type = ZCRYPT_CEX4; @@ -502,8 +501,7 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) zc->user_space_type = ZCRYPT_CEX6; } zc->min_mod_size = CEX4A_MIN_MOD_SIZE; - if (ap_test_bit(&ac->functions, AP_FUNC_MEX4K) && - ap_test_bit(&ac->functions, AP_FUNC_CRT4K)) { + if (ac->hwinfo.mex4k && ac->hwinfo.crt4k) { zc->max_mod_size = CEX4A_MAX_MOD_SIZE_4K; zc->max_exp_bit_length = CEX4A_MAX_MOD_SIZE_4K; @@ -512,7 +510,7 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) zc->max_exp_bit_length = CEX4A_MAX_MOD_SIZE_2K; } - } else if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) { + } else if (ac->hwinfo.cca) { if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) { zc->type_string = "CEX4C"; zc->speed_rating = CEX4C_SPEED_IDX; @@ -552,7 +550,7 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) zc->min_mod_size = CEX4C_MIN_MOD_SIZE; zc->max_mod_size = CEX4C_MAX_MOD_SIZE; zc->max_exp_bit_length = CEX4C_MAX_MOD_SIZE; - } else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) { + } else if (ac->hwinfo.ep11) { if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) { zc->type_string = "CEX4P"; zc->user_space_type = ZCRYPT_CEX4; @@ -595,14 +593,14 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) return rc; } - if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) { + if (ac->hwinfo.cca) { rc = sysfs_create_group(&ap_dev->device.kobj, &cca_card_attr_grp); if (rc) { zcrypt_card_unregister(zc); zcrypt_card_free(zc); } - } else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) { + } else if (ac->hwinfo.ep11) { rc = sysfs_create_group(&ap_dev->device.kobj, &ep11_card_attr_grp); if (rc) { @@ -623,9 +621,9 @@ static void zcrypt_cex4_card_remove(struct ap_device *ap_dev) struct zcrypt_card *zc = dev_get_drvdata(&ap_dev->device); struct ap_card *ac = to_ap_card(&ap_dev->device); - if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) + if (ac->hwinfo.cca) sysfs_remove_group(&ap_dev->device.kobj, &cca_card_attr_grp); - else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) + else if (ac->hwinfo.ep11) sysfs_remove_group(&ap_dev->device.kobj, &ep11_card_attr_grp); zcrypt_card_unregister(zc); @@ -650,19 +648,19 @@ static int zcrypt_cex4_queue_probe(struct ap_device *ap_dev) struct zcrypt_queue *zq; int rc; - if (ap_test_bit(&aq->card->functions, AP_FUNC_ACCEL)) { + if (aq->card->hwinfo.accel) { zq = zcrypt_queue_alloc(aq->card->maxmsgsize); if (!zq) return -ENOMEM; zq->ops = zcrypt_msgtype(MSGTYPE50_NAME, MSGTYPE50_VARIANT_DEFAULT); - } else if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) { + } else if (aq->card->hwinfo.cca) { zq = zcrypt_queue_alloc(aq->card->maxmsgsize); if (!zq) return -ENOMEM; zq->ops = zcrypt_msgtype(MSGTYPE06_NAME, MSGTYPE06_VARIANT_DEFAULT); - } else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) { + } else if (aq->card->hwinfo.ep11) { zq = zcrypt_queue_alloc(aq->card->maxmsgsize); if (!zq) return -ENOMEM; @@ -685,14 +683,14 @@ static int zcrypt_cex4_queue_probe(struct ap_device *ap_dev) return rc; } - if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) { + if (aq->card->hwinfo.cca) { rc = sysfs_create_group(&ap_dev->device.kobj, &cca_queue_attr_grp); if (rc) { zcrypt_queue_unregister(zq); zcrypt_queue_free(zq); } - } else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) { + } else if (aq->card->hwinfo.ep11) { rc = sysfs_create_group(&ap_dev->device.kobj, &ep11_queue_attr_grp); if (rc) { @@ -713,9 +711,9 @@ static void zcrypt_cex4_queue_remove(struct ap_device *ap_dev) struct zcrypt_queue *zq = dev_get_drvdata(&ap_dev->device); struct ap_queue *aq = to_ap_queue(&ap_dev->device); - if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) + if (aq->card->hwinfo.cca) sysfs_remove_group(&ap_dev->device.kobj, &cca_queue_attr_grp); - else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) + else if (aq->card->hwinfo.ep11) sysfs_remove_group(&ap_dev->device.kobj, &ep11_queue_attr_grp); zcrypt_queue_unregister(zq); diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h index 5cf88aabd64b..9a208dc4c200 100644 --- a/drivers/s390/crypto/zcrypt_debug.h +++ b/drivers/s390/crypto/zcrypt_debug.h @@ -17,7 +17,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define ZCRYPT_DBF_MAX_SPRINTF_ARGS 6 #define ZCRYPT_DBF(...) \ debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__) @@ -27,8 +27,6 @@ debug_sprintf_event(zcrypt_dbf_info, DBF_WARN, ##__VA_ARGS__) #define ZCRYPT_DBF_INFO(...) \ debug_sprintf_event(zcrypt_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define ZCRYPT_DBF_DBG(...) \ - debug_sprintf_event(zcrypt_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *zcrypt_dbf_info; diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 0a877f9792c2..3dda9589f2b9 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -6,13 +6,14 @@ * Collection of EP11 misc functions used by zcrypt and pkey */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt +#include <linux/export.h> #include <linux/init.h> +#include <linux/mempool.h> #include <linux/module.h> -#include <linux/slab.h> #include <linux/random.h> +#include <linux/slab.h> #include <asm/zcrypt.h> #include <asm/pkey.h> #include <crypto/aes.h> @@ -24,96 +25,35 @@ #include "zcrypt_ep11misc.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - #define EP11_PINBLOB_V1_BYTES 56 /* default iv used here */ static const u8 def_iv[16] = { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }; -/* ep11 card info cache */ -struct card_list_entry { - struct list_head list; - u16 cardnr; - struct ep11_card_info info; -}; -static LIST_HEAD(card_list); -static DEFINE_SPINLOCK(card_list_lock); - -static int card_cache_fetch(u16 cardnr, struct ep11_card_info *ci) -{ - int rc = -ENOENT; - struct card_list_entry *ptr; - - spin_lock_bh(&card_list_lock); - list_for_each_entry(ptr, &card_list, list) { - if (ptr->cardnr == cardnr) { - memcpy(ci, &ptr->info, sizeof(*ci)); - rc = 0; - break; - } - } - spin_unlock_bh(&card_list_lock); - - return rc; -} - -static void card_cache_update(u16 cardnr, const struct ep11_card_info *ci) -{ - int found = 0; - struct card_list_entry *ptr; - - spin_lock_bh(&card_list_lock); - list_for_each_entry(ptr, &card_list, list) { - if (ptr->cardnr == cardnr) { - memcpy(&ptr->info, ci, sizeof(*ci)); - found = 1; - break; - } - } - if (!found) { - ptr = kmalloc(sizeof(*ptr), GFP_ATOMIC); - if (!ptr) { - spin_unlock_bh(&card_list_lock); - return; - } - ptr->cardnr = cardnr; - memcpy(&ptr->info, ci, sizeof(*ci)); - list_add(&ptr->list, &card_list); - } - spin_unlock_bh(&card_list_lock); -} - -static void card_cache_scrub(u16 cardnr) -{ - struct card_list_entry *ptr; - - spin_lock_bh(&card_list_lock); - list_for_each_entry(ptr, &card_list, list) { - if (ptr->cardnr == cardnr) { - list_del(&ptr->list); - kfree(ptr); - break; - } - } - spin_unlock_bh(&card_list_lock); -} - -static void __exit card_cache_free(void) -{ - struct card_list_entry *ptr, *pnext; +/* + * Cprb memory pool held for urgent cases where no memory + * can be allocated via kmalloc. This pool is only used when + * alloc_cprbmem() is called with the xflag ZCRYPT_XFLAG_NOMEMALLOC. + */ +#define CPRB_MEMPOOL_ITEM_SIZE (8 * 1024) +static mempool_t *cprb_mempool; - spin_lock_bh(&card_list_lock); - list_for_each_entry_safe(ptr, pnext, &card_list, list) { - list_del(&ptr->list); - kfree(ptr); - } - spin_unlock_bh(&card_list_lock); -} +/* + * This is a pre-allocated memory for the device status array + * used within the ep11_findcard2() function. It is currently + * 128 * 128 * 4 bytes = 64 KB big. Usage of this memory is + * controlled via dev_status_mem_mutex. Needs adaption if more + * than 128 cards or domains to be are supported. + */ +#define ZCRYPT_DEV_STATUS_CARD_MAX 128 +#define ZCRYPT_DEV_STATUS_QUEUE_MAX 128 +#define ZCRYPT_DEV_STATUS_ENTRIES (ZCRYPT_DEV_STATUS_CARD_MAX * \ + ZCRYPT_DEV_STATUS_QUEUE_MAX) +#define ZCRYPT_DEV_STATUS_EXT_SIZE (ZCRYPT_DEV_STATUS_ENTRIES * \ + sizeof(struct zcrypt_device_status_ext)) +static void *dev_status_mem; +static DEFINE_MUTEX(dev_status_mem_mutex); static int ep11_kb_split(const u8 *kb, size_t kblen, u32 kbver, struct ep11kblob_header **kbhdr, size_t *kbhdrsize, @@ -208,7 +148,7 @@ out: * For valid ep11 keyblobs, returns a reference to the wrappingkey verification * pattern. Otherwise NULL. */ -const u8 *ep11_kb_wkvp(const u8 *keyblob, size_t keybloblen) +const u8 *ep11_kb_wkvp(const u8 *keyblob, u32 keybloblen) { struct ep11keyblob *kb; @@ -222,7 +162,7 @@ EXPORT_SYMBOL(ep11_kb_wkvp); * Simple check if the key blob is a valid EP11 AES key blob with header. */ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11kblob_header *hdr = (struct ep11kblob_header *)key; struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr)); @@ -230,7 +170,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*hdr) + sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*hdr) + sizeof(*kb)); return -EINVAL; } @@ -255,7 +195,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, } if (hdr->len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)hdr->len, keylen); return -EINVAL; } @@ -289,7 +229,7 @@ EXPORT_SYMBOL(ep11_check_aes_key_with_hdr); * Simple check if the key blob is a valid EP11 ECC key blob with header. */ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11kblob_header *hdr = (struct ep11kblob_header *)key; struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr)); @@ -297,7 +237,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*hdr) + sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*hdr) + sizeof(*kb)); return -EINVAL; } @@ -322,7 +262,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, } if (hdr->len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)hdr->len, keylen); return -EINVAL; } @@ -357,14 +297,14 @@ EXPORT_SYMBOL(ep11_check_ecc_key_with_hdr); * the header in the session field (old style EP11 AES key). */ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11keyblob *kb = (struct ep11keyblob *)key; #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*kb)); return -EINVAL; } @@ -383,7 +323,7 @@ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, } if (kb->head.len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)kb->head.len, keylen); return -EINVAL; } @@ -416,14 +356,20 @@ EXPORT_SYMBOL(ep11_check_aes_key); /* * Allocate and prepare ep11 cprb plus additional payload. */ -static inline struct ep11_cprb *alloc_cprb(size_t payload_len) +static void *alloc_cprbmem(size_t payload_len, u32 xflags) { size_t len = sizeof(struct ep11_cprb) + payload_len; - struct ep11_cprb *cprb; + struct ep11_cprb *cprb = NULL; - cprb = kzalloc(len, GFP_KERNEL); + if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) { + if (len <= CPRB_MEMPOOL_ITEM_SIZE) + cprb = mempool_alloc_preallocated(cprb_mempool); + } else { + cprb = kmalloc(len, GFP_KERNEL); + } if (!cprb) return NULL; + memset(cprb, 0, len); cprb->cprb_len = sizeof(struct ep11_cprb); cprb->cprb_ver_id = 0x04; @@ -435,6 +381,20 @@ static inline struct ep11_cprb *alloc_cprb(size_t payload_len) } /* + * Free ep11 cprb buffer space. + */ +static void free_cprbmem(void *mem, size_t payload_len, bool scrub, u32 xflags) +{ + if (mem && scrub) + memzero_explicit(mem, sizeof(struct ep11_cprb) + payload_len); + + if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) + mempool_free(mem, cprb_mempool); + else + kfree(mem); +} + +/* * Some helper functions related to ASN1 encoding. * Limited to length info <= 2 byte. */ @@ -494,6 +454,7 @@ static inline void prep_urb(struct ep11_urb *u, struct ep11_cprb *req, size_t req_len, struct ep11_cprb *rep, size_t rep_len) { + memset(u, 0, sizeof(*u)); u->targets = (u8 __user *)t; u->targets_num = nt; u->req = (u8 __user *)req; @@ -510,7 +471,7 @@ static int check_reply_pl(const u8 *pl, const char *func) /* start tag */ if (*pl++ != 0x30) { - DEBUG_ERR("%s reply start tag mismatch\n", func); + ZCRYPT_DBF_ERR("%s reply start tag mismatch\n", func); return -EIO; } @@ -527,51 +488,68 @@ static int check_reply_pl(const u8 *pl, const char *func) len = *((u16 *)pl); pl += 2; } else { - DEBUG_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", - func, *pl); + ZCRYPT_DBF_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", + func, *pl); return -EIO; } /* len should cover at least 3 fields with 32 bit value each */ if (len < 3 * 6) { - DEBUG_ERR("%s reply length %d too small\n", func, len); + ZCRYPT_DBF_ERR("%s reply length %d too small\n", func, len); return -EIO; } /* function tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s function tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s function tag or length mismatch\n", func); return -EIO; } pl += 6; /* dom tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s dom tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s dom tag or length mismatch\n", func); return -EIO; } pl += 6; /* return value tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s return value tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s return value tag or length mismatch\n", + func); return -EIO; } pl += 2; ret = *((u32 *)pl); if (ret != 0) { - DEBUG_ERR("%s return value 0x%04x != 0\n", func, ret); + ZCRYPT_DBF_ERR("%s return value 0x%08x != 0\n", func, ret); return -EIO; } return 0; } +/* Check ep11 reply cprb, return 0 or suggested errno value. */ +static int check_reply_cprb(const struct ep11_cprb *rep, const char *func) +{ + /* check ep11 reply return code field */ + if (rep->ret_code) { + ZCRYPT_DBF_ERR("%s ep11 reply ret_code=0x%08x\n", __func__, + rep->ret_code); + if (rep->ret_code == 0x000c0003) + return -EBUSY; + else + return -EIO; + } + + return 0; +} + /* * Helper function which does an ep11 query with given query type. */ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, - size_t buflen, u8 *buf) + size_t buflen, u8 *buf, u32 xflags) { struct ep11_info_req_pl { struct pl_head head; @@ -593,11 +571,11 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, } __packed * rep_pl; struct ep11_cprb *req = NULL, *rep = NULL; struct ep11_target_dev target; - struct ep11_urb *urb = NULL; + struct ep11_urb urb; int api = EP11_API_V1, rc = -ENOMEM; /* request cprb and payload */ - req = alloc_cprb(sizeof(struct ep11_info_req_pl)); + req = alloc_cprbmem(sizeof(struct ep11_info_req_pl), xflags); if (!req) goto out; req_pl = (struct ep11_info_req_pl *)(((u8 *)req) + sizeof(*req)); @@ -609,40 +587,42 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, req_pl->query_subtype_len = sizeof(u32); /* reply cprb and payload */ - rep = alloc_cprb(sizeof(struct ep11_info_rep_pl) + buflen); + rep = alloc_cprbmem(sizeof(struct ep11_info_rep_pl) + buflen, xflags); if (!rep) goto out; rep_pl = (struct ep11_info_rep_pl *)(((u8 *)rep) + sizeof(*rep)); /* urb and target */ - urb = kmalloc(sizeof(*urb), GFP_KERNEL); - if (!urb) - goto out; target.ap_id = cardnr; target.dom_id = domain; - prep_urb(urb, &target, 1, + prep_urb(&urb, &target, 1, req, sizeof(*req) + sizeof(*req_pl), rep, sizeof(*rep) + sizeof(*rep_pl) + buflen); - rc = zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(&urb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } + /* check ep11 reply cprb */ + rc = check_reply_cprb(rep, __func__); + if (rc) + goto out; + + /* check payload */ rc = check_reply_pl((u8 *)rep_pl, __func__); if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > buflen) { - DEBUG_ERR("%s mismatch between reply data len and buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch between reply data len and buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -650,16 +630,15 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, memcpy(buf, ((u8 *)rep_pl) + sizeof(*rep_pl), rep_pl->data_len); out: - kfree(req); - kfree(rep); - kfree(urb); + free_cprbmem(req, 0, false, xflags); + free_cprbmem(rep, 0, false, xflags); return rc; } /* * Provide information about an EP11 card. */ -int ep11_get_card_info(u16 card, struct ep11_card_info *info, int verify) +int ep11_get_card_info(u16 card, struct ep11_card_info *info, u32 xflags) { int rc; struct ep11_module_query_info { @@ -689,30 +668,26 @@ int ep11_get_card_info(u16 card, struct ep11_card_info *info, int verify) u32 max_CP_index; } __packed * pmqi = NULL; - rc = card_cache_fetch(card, info); - if (rc || verify) { - pmqi = kmalloc(sizeof(*pmqi), GFP_KERNEL); - if (!pmqi) - return -ENOMEM; - rc = ep11_query_info(card, AUTOSEL_DOM, - 0x01 /* module info query */, - sizeof(*pmqi), (u8 *)pmqi); - if (rc) { - if (rc == -ENODEV) - card_cache_scrub(card); - goto out; - } - memset(info, 0, sizeof(*info)); - info->API_ord_nr = pmqi->API_ord_nr; - info->FW_version = - (pmqi->FW_major_vers << 8) + pmqi->FW_minor_vers; - memcpy(info->serial, pmqi->serial, sizeof(info->serial)); - info->op_mode = pmqi->op_mode; - card_cache_update(card, info); - } + /* use the cprb mempool to satisfy this short term mem alloc */ + pmqi = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ? + mempool_alloc_preallocated(cprb_mempool) : + mempool_alloc(cprb_mempool, GFP_KERNEL); + if (!pmqi) + return -ENOMEM; + rc = ep11_query_info(card, AUTOSEL_DOM, + 0x01 /* module info query */, + sizeof(*pmqi), (u8 *)pmqi, xflags); + if (rc) + goto out; + + memset(info, 0, sizeof(*info)); + info->API_ord_nr = pmqi->API_ord_nr; + info->FW_version = (pmqi->FW_major_vers << 8) + pmqi->FW_minor_vers; + memcpy(info->serial, pmqi->serial, sizeof(info->serial)); + info->op_mode = pmqi->op_mode; out: - kfree(pmqi); + mempool_free(pmqi, cprb_mempool); return rc; } EXPORT_SYMBOL(ep11_get_card_info); @@ -720,7 +695,8 @@ EXPORT_SYMBOL(ep11_get_card_info); /* * Provide information about a domain within an EP11 card. */ -int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info) +int ep11_get_domain_info(u16 card, u16 domain, + struct ep11_domain_info *info, u32 xflags) { int rc; struct ep11_domain_query_info { @@ -729,36 +705,32 @@ int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info) u8 new_WK_VP[32]; u32 dom_flags; u64 op_mode; - } __packed * p_dom_info; - - p_dom_info = kmalloc(sizeof(*p_dom_info), GFP_KERNEL); - if (!p_dom_info) - return -ENOMEM; + } __packed dom_query_info; rc = ep11_query_info(card, domain, 0x03 /* domain info query */, - sizeof(*p_dom_info), (u8 *)p_dom_info); + sizeof(dom_query_info), (u8 *)&dom_query_info, + xflags); if (rc) goto out; memset(info, 0, sizeof(*info)); info->cur_wk_state = '0'; info->new_wk_state = '0'; - if (p_dom_info->dom_flags & 0x10 /* left imprint mode */) { - if (p_dom_info->dom_flags & 0x02 /* cur wk valid */) { + if (dom_query_info.dom_flags & 0x10 /* left imprint mode */) { + if (dom_query_info.dom_flags & 0x02 /* cur wk valid */) { info->cur_wk_state = '1'; - memcpy(info->cur_wkvp, p_dom_info->cur_WK_VP, 32); + memcpy(info->cur_wkvp, dom_query_info.cur_WK_VP, 32); } - if (p_dom_info->dom_flags & 0x04 || /* new wk present */ - p_dom_info->dom_flags & 0x08 /* new wk committed */) { + if (dom_query_info.dom_flags & 0x04 || /* new wk present */ + dom_query_info.dom_flags & 0x08 /* new wk committed */) { info->new_wk_state = - p_dom_info->dom_flags & 0x08 ? '2' : '1'; - memcpy(info->new_wkvp, p_dom_info->new_WK_VP, 32); + dom_query_info.dom_flags & 0x08 ? '2' : '1'; + memcpy(info->new_wkvp, dom_query_info.new_WK_VP, 32); } } - info->op_mode = p_dom_info->op_mode; + info->op_mode = dom_query_info.op_mode; out: - kfree(p_dom_info); return rc; } EXPORT_SYMBOL(ep11_get_domain_info); @@ -771,7 +743,7 @@ EXPORT_SYMBOL(ep11_get_domain_info); static int _ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize) + u8 *keybuf, size_t *keybufsize, u32 xflags) { struct keygen_req_pl { struct pl_head head; @@ -806,7 +778,7 @@ static int _ep11_genaeskey(u16 card, u16 domain, struct ep11_cprb *req = NULL, *rep = NULL; size_t req_pl_size, pinblob_size = 0; struct ep11_target_dev target; - struct ep11_urb *urb = NULL; + struct ep11_urb urb; int api, rc = -ENOMEM; u8 *p; @@ -816,9 +788,8 @@ static int _ep11_genaeskey(u16 card, u16 domain, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -835,7 +806,7 @@ static int _ep11_genaeskey(u16 card, u16 domain, pinblob_size = EP11_PINBLOB_V1_BYTES; } req_pl_size = sizeof(struct keygen_req_pl) + ASN1TAGLEN(pinblob_size); - req = alloc_cprb(req_pl_size); + req = alloc_cprbmem(req_pl_size, xflags); if (!req) goto out; req_pl = (struct keygen_req_pl *)(((u8 *)req) + sizeof(*req)); @@ -861,40 +832,42 @@ static int _ep11_genaeskey(u16 card, u16 domain, *p++ = pinblob_size; /* reply cprb and payload */ - rep = alloc_cprb(sizeof(struct keygen_rep_pl)); + rep = alloc_cprbmem(sizeof(struct keygen_rep_pl), xflags); if (!rep) goto out; rep_pl = (struct keygen_rep_pl *)(((u8 *)rep) + sizeof(*rep)); /* urb and target */ - urb = kmalloc(sizeof(*urb), GFP_KERNEL); - if (!urb) - goto out; target.ap_id = card; target.dom_id = domain; - prep_urb(urb, &target, 1, + prep_urb(&urb, &target, 1, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(&urb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } + /* check ep11 reply cprb */ + rc = check_reply_cprb(rep, __func__); + if (rc) + goto out; + + /* check payload */ rc = check_reply_pl((u8 *)rep_pl, __func__); if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -904,14 +877,13 @@ static int _ep11_genaeskey(u16 card, u16 domain, *keybufsize = rep_pl->data_len; out: - kfree(req); - kfree(rep); - kfree(urb); + free_cprbmem(req, 0, false, xflags); + free_cprbmem(rep, sizeof(struct keygen_rep_pl), true, xflags); return rc; } int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, u32 keybufver) + u8 *keybuf, u32 *keybufsize, u32 keybufver, u32 xflags) { struct ep11kblob_header *hdr; size_t hdr_size, pl_size; @@ -932,7 +904,7 @@ int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, return rc; rc = _ep11_genaeskey(card, domain, keybitsize, keygenflags, - pl, &pl_size); + pl, &pl_size, xflags); if (rc) return rc; @@ -952,7 +924,8 @@ static int ep11_cryptsingle(u16 card, u16 domain, u16 mode, u32 mech, const u8 *iv, const u8 *key, size_t keysize, const u8 *inbuf, size_t inbufsize, - u8 *outbuf, size_t *outbufsize) + u8 *outbuf, size_t *outbufsize, + u32 xflags) { struct crypt_req_pl { struct pl_head head; @@ -979,8 +952,8 @@ static int ep11_cryptsingle(u16 card, u16 domain, } __packed * rep_pl; struct ep11_cprb *req = NULL, *rep = NULL; struct ep11_target_dev target; - struct ep11_urb *urb = NULL; - size_t req_pl_size, rep_pl_size; + struct ep11_urb urb; + size_t req_pl_size, rep_pl_size = 0; int n, api = EP11_API_V1, rc = -ENOMEM; u8 *p; @@ -991,7 +964,7 @@ static int ep11_cryptsingle(u16 card, u16 domain, /* request cprb and payload */ req_pl_size = sizeof(struct crypt_req_pl) + (iv ? 16 : 0) + ASN1TAGLEN(keysize) + ASN1TAGLEN(inbufsize); - req = alloc_cprb(req_pl_size); + req = alloc_cprbmem(req_pl_size, xflags); if (!req) goto out; req_pl = (struct crypt_req_pl *)(((u8 *)req) + sizeof(*req)); @@ -1013,34 +986,36 @@ static int ep11_cryptsingle(u16 card, u16 domain, /* reply cprb and payload, assume out data size <= in data size + 32 */ rep_pl_size = sizeof(struct crypt_rep_pl) + ASN1TAGLEN(inbufsize + 32); - rep = alloc_cprb(rep_pl_size); + rep = alloc_cprbmem(rep_pl_size, xflags); if (!rep) goto out; rep_pl = (struct crypt_rep_pl *)(((u8 *)rep) + sizeof(*rep)); /* urb and target */ - urb = kmalloc(sizeof(*urb), GFP_KERNEL); - if (!urb) - goto out; target.ap_id = card; target.dom_id = domain; - prep_urb(urb, &target, 1, + prep_urb(&urb, &target, 1, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + rep_pl_size); - rc = zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(&urb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } + /* check ep11 reply cprb */ + rc = check_reply_cprb(rep, __func__); + if (rc) + goto out; + + /* check payload */ rc = check_reply_pl((u8 *)rep_pl, __func__); if (rc) goto out; if (rep_pl->data_tag != 0x04) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } @@ -1053,14 +1028,14 @@ static int ep11_cryptsingle(u16 card, u16 domain, n = *((u16 *)p); p += 2; } else { - DEBUG_ERR("%s unknown reply data length format 0x%02hhx\n", - __func__, rep_pl->data_lenfmt); + ZCRYPT_DBF_ERR("%s unknown reply data length format 0x%02hhx\n", + __func__, rep_pl->data_lenfmt); rc = -EIO; goto out; } if (n > *outbufsize) { - DEBUG_ERR("%s mismatch reply data len %d / output buffer %zu\n", - __func__, n, *outbufsize); + ZCRYPT_DBF_ERR("%s mismatch reply data len %d / output buffer %zu\n", + __func__, n, *outbufsize); rc = -ENOSPC; goto out; } @@ -1069,9 +1044,8 @@ static int ep11_cryptsingle(u16 card, u16 domain, *outbufsize = n; out: - kfree(req); - kfree(rep); - kfree(urb); + free_cprbmem(req, req_pl_size, true, xflags); + free_cprbmem(rep, rep_pl_size, true, xflags); return rc; } @@ -1080,7 +1054,7 @@ static int _ep11_unwrapkey(u16 card, u16 domain, const u8 *enckey, size_t enckeysize, u32 mech, const u8 *iv, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize) + u8 *keybuf, size_t *keybufsize, u32 xflags) { struct uw_req_pl { struct pl_head head; @@ -1117,7 +1091,7 @@ static int _ep11_unwrapkey(u16 card, u16 domain, struct ep11_cprb *req = NULL, *rep = NULL; size_t req_pl_size, pinblob_size = 0; struct ep11_target_dev target; - struct ep11_urb *urb = NULL; + struct ep11_urb urb; int api, rc = -ENOMEM; u8 *p; @@ -1135,7 +1109,7 @@ static int _ep11_unwrapkey(u16 card, u16 domain, req_pl_size = sizeof(struct uw_req_pl) + (iv ? 16 : 0) + ASN1TAGLEN(keksize) + ASN1TAGLEN(0) + ASN1TAGLEN(pinblob_size) + ASN1TAGLEN(enckeysize); - req = alloc_cprb(req_pl_size); + req = alloc_cprbmem(req_pl_size, xflags); if (!req) goto out; req_pl = (struct uw_req_pl *)(((u8 *)req) + sizeof(*req)); @@ -1171,40 +1145,42 @@ static int _ep11_unwrapkey(u16 card, u16 domain, p += asn1tag_write(p, 0x04, enckey, enckeysize); /* reply cprb and payload */ - rep = alloc_cprb(sizeof(struct uw_rep_pl)); + rep = alloc_cprbmem(sizeof(struct uw_rep_pl), xflags); if (!rep) goto out; rep_pl = (struct uw_rep_pl *)(((u8 *)rep) + sizeof(*rep)); /* urb and target */ - urb = kmalloc(sizeof(*urb), GFP_KERNEL); - if (!urb) - goto out; target.ap_id = card; target.dom_id = domain; - prep_urb(urb, &target, 1, + prep_urb(&urb, &target, 1, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(&urb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } + /* check ep11 reply cprb */ + rc = check_reply_cprb(rep, __func__); + if (rc) + goto out; + + /* check payload */ rc = check_reply_pl((u8 *)rep_pl, __func__); if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1214,9 +1190,8 @@ static int _ep11_unwrapkey(u16 card, u16 domain, *keybufsize = rep_pl->data_len; out: - kfree(req); - kfree(rep); - kfree(urb); + free_cprbmem(req, req_pl_size, true, xflags); + free_cprbmem(rep, sizeof(struct uw_rep_pl), true, xflags); return rc; } @@ -1225,8 +1200,8 @@ static int ep11_unwrapkey(u16 card, u16 domain, const u8 *enckey, size_t enckeysize, u32 mech, const u8 *iv, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, - u8 keybufver) + u8 *keybuf, u32 *keybufsize, + u8 keybufver, u32 xflags) { struct ep11kblob_header *hdr; size_t hdr_size, pl_size; @@ -1240,7 +1215,7 @@ static int ep11_unwrapkey(u16 card, u16 domain, rc = _ep11_unwrapkey(card, domain, kek, keksize, enckey, enckeysize, mech, iv, keybitsize, keygenflags, - pl, &pl_size); + pl, &pl_size, xflags); if (rc) return rc; @@ -1259,7 +1234,7 @@ static int ep11_unwrapkey(u16 card, u16 domain, static int _ep11_wrapkey(u16 card, u16 domain, const u8 *key, size_t keysize, u32 mech, const u8 *iv, - u8 *databuf, size_t *datasize) + u8 *databuf, size_t *datasize, u32 xflags) { struct wk_req_pl { struct pl_head head; @@ -1288,7 +1263,7 @@ static int _ep11_wrapkey(u16 card, u16 domain, } __packed * rep_pl; struct ep11_cprb *req = NULL, *rep = NULL; struct ep11_target_dev target; - struct ep11_urb *urb = NULL; + struct ep11_urb urb; size_t req_pl_size; int api, rc = -ENOMEM; u8 *p; @@ -1296,7 +1271,7 @@ static int _ep11_wrapkey(u16 card, u16 domain, /* request cprb and payload */ req_pl_size = sizeof(struct wk_req_pl) + (iv ? 16 : 0) + ASN1TAGLEN(keysize) + 4; - req = alloc_cprb(req_pl_size); + req = alloc_cprbmem(req_pl_size, xflags); if (!req) goto out; if (!mech || mech == 0x80060001) @@ -1326,40 +1301,42 @@ static int _ep11_wrapkey(u16 card, u16 domain, *p++ = 0; /* reply cprb and payload */ - rep = alloc_cprb(sizeof(struct wk_rep_pl)); + rep = alloc_cprbmem(sizeof(struct wk_rep_pl), xflags); if (!rep) goto out; rep_pl = (struct wk_rep_pl *)(((u8 *)rep) + sizeof(*rep)); /* urb and target */ - urb = kmalloc(sizeof(*urb), GFP_KERNEL); - if (!urb) - goto out; target.ap_id = card; target.dom_id = domain; - prep_urb(urb, &target, 1, + prep_urb(&urb, &target, 1, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(&urb, xflags); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } + /* check ep11 reply cprb */ + rc = check_reply_cprb(rep, __func__); + if (rc) + goto out; + + /* check payload */ rc = check_reply_pl((u8 *)rep_pl, __func__); if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *datasize) { - DEBUG_ERR("%s mismatch reply data len / data buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / data buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1369,78 +1346,83 @@ static int _ep11_wrapkey(u16 card, u16 domain, *datasize = rep_pl->data_len; out: - kfree(req); - kfree(rep); - kfree(urb); + free_cprbmem(req, req_pl_size, true, xflags); + free_cprbmem(rep, sizeof(struct wk_rep_pl), true, xflags); return rc; } int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize, - u32 keytype) + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, + u32 keytype, u32 xflags) { int rc; - u8 encbuf[64], *kek = NULL; + void *mem; + u8 encbuf[64], *kek; size_t clrkeylen, keklen, encbuflen = sizeof(encbuf); if (keybitsize == 128 || keybitsize == 192 || keybitsize == 256) { clrkeylen = keybitsize / 8; } else { - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); return -EINVAL; } - /* allocate memory for the temp kek */ + /* + * Allocate space for the temp kek. + * Also we only need up to MAXEP11AESKEYBLOBSIZE bytes for this + * we use the already existing cprb mempool to solve this + * short term memory requirement. + */ + mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ? + mempool_alloc_preallocated(cprb_mempool) : + mempool_alloc(cprb_mempool, GFP_KERNEL); + if (!mem) + return -ENOMEM; + kek = (u8 *)mem; keklen = MAXEP11AESKEYBLOBSIZE; - kek = kmalloc(keklen, GFP_ATOMIC); - if (!kek) { - rc = -ENOMEM; - goto out; - } /* Step 1: generate AES 256 bit random kek key */ rc = _ep11_genaeskey(card, domain, 256, 0x00006c00, /* EN/DECRYPT, WRAP/UNWRAP */ - kek, &keklen); + kek, &keklen, xflags); if (rc) { - DEBUG_ERR( - "%s generate kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s generate kek key failed, rc=%d\n", + __func__, rc); goto out; } /* Step 2: encrypt clear key value with the kek key */ rc = ep11_cryptsingle(card, domain, 0, 0, def_iv, kek, keklen, - clrkey, clrkeylen, encbuf, &encbuflen); + clrkey, clrkeylen, encbuf, &encbuflen, xflags); if (rc) { - DEBUG_ERR( - "%s encrypting key value with kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s encrypting key value with kek key failed, rc=%d\n", + __func__, rc); goto out; } /* Step 3: import the encrypted key value as a new key */ rc = ep11_unwrapkey(card, domain, kek, keklen, encbuf, encbuflen, 0, def_iv, - keybitsize, 0, keybuf, keybufsize, keytype); + keybitsize, keygenflags, + keybuf, keybufsize, + keytype, xflags); if (rc) { - DEBUG_ERR( - "%s importing key value as new key failed,, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s importing key value as new key failed, rc=%d\n", + __func__, rc); goto out; } out: - kfree(kek); + mempool_free(mem, cprb_mempool); return rc; } EXPORT_SYMBOL(ep11_clr2keyblob); int ep11_kblob2protkey(u16 card, u16 dom, - const u8 *keyblob, size_t keybloblen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) + const u8 *keyblob, u32 keybloblen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags) { struct ep11kblob_header *hdr; struct ep11keyblob *key; @@ -1466,27 +1448,40 @@ int ep11_kblob2protkey(u16 card, u16 dom, } /* !!! hdr is no longer a valid header !!! */ - /* alloc temp working buffer */ + /* need a temp working buffer */ wkbuflen = (keylen + AES_BLOCK_SIZE) & (~(AES_BLOCK_SIZE - 1)); - wkbuf = kmalloc(wkbuflen, GFP_ATOMIC); - if (!wkbuf) - return -ENOMEM; + if (wkbuflen > CPRB_MEMPOOL_ITEM_SIZE) { + /* this should never happen */ + rc = -ENOMEM; + ZCRYPT_DBF_WARN("%s wkbuflen %d > cprb mempool item size %d, rc=%d\n", + __func__, (int)wkbuflen, CPRB_MEMPOOL_ITEM_SIZE, rc); + return rc; + } + /* use the cprb mempool to satisfy this short term mem allocation */ + wkbuf = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ? + mempool_alloc_preallocated(cprb_mempool) : + mempool_alloc(cprb_mempool, GFP_ATOMIC); + if (!wkbuf) { + rc = -ENOMEM; + ZCRYPT_DBF_WARN("%s allocating tmp buffer via cprb mempool failed, rc=%d\n", + __func__, rc); + return rc; + } /* ep11 secure key -> protected key + info */ rc = _ep11_wrapkey(card, dom, (u8 *)key, keylen, - 0, def_iv, wkbuf, &wkbuflen); + 0, def_iv, wkbuf, &wkbuflen, xflags); if (rc) { - DEBUG_ERR( - "%s rewrapping ep11 key to pkey failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s rewrapping ep11 key to pkey failed, rc=%d\n", + __func__, rc); goto out; } wki = (struct wk_info *)wkbuf; /* check struct version and pkey type */ if (wki->version != 1 || wki->pkeytype < 1 || wki->pkeytype > 5) { - DEBUG_ERR("%s wk info version %d or pkeytype %d mismatch.\n", - __func__, (int)wki->version, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s wk info version %d or pkeytype %d mismatch.\n", + __func__, (int)wki->version, (int)wki->pkeytype); rc = -EIO; goto out; } @@ -1511,8 +1506,8 @@ int ep11_kblob2protkey(u16 card, u16 dom, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported AES pkeysize %d\n", - __func__, (int)wki->pkeysize); + ZCRYPT_DBF_ERR("%s unknown/unsupported AES pkeysize %d\n", + __func__, (int)wki->pkeysize); rc = -EIO; goto out; } @@ -1525,16 +1520,16 @@ int ep11_kblob2protkey(u16 card, u16 dom, break; case 2: /* TDES */ default: - DEBUG_ERR("%s unknown/unsupported key type %d\n", - __func__, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, (int)wki->pkeytype); rc = -EIO; goto out; } /* copy the translated protected key */ if (wki->pkeysize > *protkeylen) { - DEBUG_ERR("%s wk info pkeysize %llu > protkeysize %u\n", - __func__, wki->pkeysize, *protkeylen); + ZCRYPT_DBF_ERR("%s wk info pkeysize %llu > protkeysize %u\n", + __func__, wki->pkeysize, *protkeylen); rc = -EINVAL; goto out; } @@ -1542,37 +1537,32 @@ int ep11_kblob2protkey(u16 card, u16 dom, *protkeylen = wki->pkeysize; out: - kfree(wkbuf); + mempool_free(wkbuf, cprb_mempool); return rc; } EXPORT_SYMBOL(ep11_kblob2protkey); -int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, - int minhwtype, int minapi, const u8 *wkvp) +int ep11_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain, + int minhwtype, int minapi, const u8 *wkvp, u32 xflags) { struct zcrypt_device_status_ext *device_status; - u32 *_apqns = NULL, _nr_apqns = 0; - int i, card, dom, rc = -ENOMEM; struct ep11_domain_info edi; struct ep11_card_info eci; + u32 _nr_apqns = 0; + int i, card, dom; - /* fetch status of all crypto cards */ - device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT, - sizeof(struct zcrypt_device_status_ext), - GFP_KERNEL); - if (!device_status) - return -ENOMEM; - zcrypt_device_status_mask_ext(device_status); + /* occupy the device status memory */ + mutex_lock(&dev_status_mem_mutex); + memset(dev_status_mem, 0, ZCRYPT_DEV_STATUS_EXT_SIZE); + device_status = (struct zcrypt_device_status_ext *)dev_status_mem; - /* allocate 1k space for up to 256 apqns */ - _apqns = kmalloc_array(256, sizeof(u32), GFP_KERNEL); - if (!_apqns) { - kvfree(device_status); - return -ENOMEM; - } + /* fetch crypto device status into this struct */ + zcrypt_device_status_mask_ext(device_status, + ZCRYPT_DEV_STATUS_CARD_MAX, + ZCRYPT_DEV_STATUS_QUEUE_MAX); /* walk through all the crypto apqnss */ - for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { + for (i = 0; i < ZCRYPT_DEV_STATUS_ENTRIES; i++) { card = AP_QID_CARD(device_status[i].qid); dom = AP_QID_QUEUE(device_status[i].qid); /* check online state */ @@ -1592,14 +1582,14 @@ int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, continue; /* check min api version if given */ if (minapi > 0) { - if (ep11_get_card_info(card, &eci, 0)) + if (ep11_get_card_info(card, &eci, xflags)) continue; if (minapi > eci.API_ord_nr) continue; } /* check wkvp if given */ if (wkvp) { - if (ep11_get_domain_info(card, dom, &edi)) + if (ep11_get_domain_info(card, dom, &edi, xflags)) continue; if (edi.cur_wk_state != '1') continue; @@ -1607,27 +1597,40 @@ int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, continue; } /* apqn passed all filtering criterons, add to the array */ - if (_nr_apqns < 256) - _apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom); + if (_nr_apqns < *nr_apqns) + apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom); } - /* nothing found ? */ - if (!_nr_apqns) { - kfree(_apqns); - rc = -ENODEV; - } else { - /* no re-allocation, simple return the _apqns array */ - *apqns = _apqns; - *nr_apqns = _nr_apqns; - rc = 0; - } + *nr_apqns = _nr_apqns; - kvfree(device_status); - return rc; + mutex_unlock(&dev_status_mem_mutex); + + return _nr_apqns ? 0 : -ENODEV; } EXPORT_SYMBOL(ep11_findcard2); -void __exit zcrypt_ep11misc_exit(void) +int __init zcrypt_ep11misc_init(void) +{ + /* Pre-allocate a small memory pool for ep11 cprbs. */ + cprb_mempool = mempool_create_kmalloc_pool(2 * zcrypt_mempool_threshold, + CPRB_MEMPOOL_ITEM_SIZE); + if (!cprb_mempool) + return -ENOMEM; + + /* Pre-allocate one crypto status card struct used in ep11_findcard2() */ + dev_status_mem = kvmalloc(ZCRYPT_DEV_STATUS_EXT_SIZE, GFP_KERNEL); + if (!dev_status_mem) { + mempool_destroy(cprb_mempool); + return -ENOMEM; + } + + return 0; +} + +void zcrypt_ep11misc_exit(void) { - card_cache_free(); + mutex_lock(&dev_status_mem_mutex); + kvfree(dev_status_mem); + mutex_unlock(&dev_status_mem_mutex); + mempool_destroy(cprb_mempool); } diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h index 9d17fd5228a7..b5e6fd861815 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.h +++ b/drivers/s390/crypto/zcrypt_ep11misc.h @@ -54,7 +54,7 @@ static inline bool is_ep11_keyblob(const u8 *key) * For valid ep11 keyblobs, returns a reference to the wrappingkey verification * pattern. Otherwise NULL. */ -const u8 *ep11_kb_wkvp(const u8 *kblob, size_t kbloblen); +const u8 *ep11_kb_wkvp(const u8 *kblob, u32 kbloblen); /* * Simple check if the key blob is a valid EP11 AES key blob with header. @@ -63,7 +63,7 @@ const u8 *ep11_kb_wkvp(const u8 *kblob, size_t kbloblen); * Returns 0 on success or errno value on failure. */ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* * Simple check if the key blob is a valid EP11 ECC key blob with header. @@ -72,7 +72,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, * Returns 0 on success or errno value on failure. */ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* * Simple check if the key blob is a valid EP11 AES key blob with @@ -82,7 +82,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, * Returns 0 on success or errno value on failure. */ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* EP11 card info struct */ struct ep11_card_info { @@ -104,25 +104,26 @@ struct ep11_domain_info { /* * Provide information about an EP11 card. */ -int ep11_get_card_info(u16 card, struct ep11_card_info *info, int verify); +int ep11_get_card_info(u16 card, struct ep11_card_info *info, u32 xflags); /* * Provide information about a domain within an EP11 card. */ -int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info); +int ep11_get_domain_info(u16 card, u16 domain, + struct ep11_domain_info *info, u32 xflags); /* * Generate (random) EP11 AES secure key. */ int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, u32 keybufver); + u8 *keybuf, u32 *keybufsize, u32 keybufver, u32 xflags); /* * Generate EP11 AES secure key with given clear key value. */ int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize, - u32 keytype); + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, + u32 keytype, u32 xflags); /* * Build a list of ep11 apqns meeting the following constrains: @@ -136,22 +137,22 @@ int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, * key for this domain. When a wkvp is given there will always be a re-fetch * of the domain info for the potential apqn - so this triggers an request * reply to each apqn eligible. - * The array of apqn entries is allocated with kmalloc and returned in *apqns; - * the number of apqns stored into the list is returned in *nr_apqns. One apqn - * entry is simple a 32 bit value with 16 bit cardnr and 16 bit domain nr and - * may be casted to struct pkey_apqn. The return value is either 0 for success - * or a negative errno value. If no apqn meeting the criteria is found, - * -ENODEV is returned. + * The caller should set *nr_apqns to the nr of elements available in *apqns. + * On return *nr_apqns is then updated with the nr of apqns filled into *apqns. + * The return value is either 0 for success or a negative errno value. + * If no apqn meeting the criteria is found, -ENODEV is returned. */ -int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, - int minhwtype, int minapi, const u8 *wkvp); +int ep11_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain, + int minhwtype, int minapi, const u8 *wkvp, u32 xflags); /* * Derive proteced key from EP11 key blob (AES and ECC keys). */ -int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype, + u32 xflags); +int zcrypt_ep11misc_init(void); void zcrypt_ep11misc_exit(void); #endif /* _ZCRYPT_EP11MISC_H_ */ diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index d36177e65a3d..46e27b43a8af 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -98,17 +98,30 @@ static inline int convert_error(struct zcrypt_queue *zq, case REP88_ERROR_MESSAGE_MALFORMD: /* 0x22 */ case REP88_ERROR_KEY_TYPE: /* 0x34 */ /* RY indicates malformed request */ - ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", - __func__, card, queue, ehdr->reply_code); + if (ehdr->reply_code == REP82_ERROR_FILTERED_BY_HYPERVISOR && + ehdr->type == TYPE86_RSP_CODE) { + struct { + struct type86_hdr hdr; + struct type86_fmt2_ext fmt2; + } __packed * head = reply->msg; + unsigned int apfs = *((u32 *)head->fmt2.apfs); + + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x apfs=0x%x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code, apfs); + } else { + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code); + } return -EINVAL; case REP82_ERROR_MACHINE_FAILURE: /* 0x10 */ case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ case REP82_ERROR_TRANSPORT_FAIL: /* 0x90 */ /* - * Msg to wrong type or card/infrastructure failure. - * Trigger rescan of the ap bus, trigger retry request. + * Msg to wrong type or card/infrastructure failure. Return + * EAGAIN, the upper layer may do a retry on the request. */ - atomic_set(&zcrypt_rescan_req, 1); /* For type 86 response show the apfs value (failure reason) */ if (ehdr->reply_code == REP82_ERROR_TRANSPORT_FAIL && ehdr->type == TYPE86_RSP_CODE) { diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index 2e155de8abe5..d6fc2d8e7fad 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -10,8 +10,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/module.h> #include <linux/slab.h> @@ -427,7 +426,7 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, len = t80h->len; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -438,7 +437,7 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, msg->len = sizeof(error_reply); } out: - complete((struct completion *)msg->private); + complete(&msg->response.work); } static atomic_t zcrypt_step = ATOMIC_INIT(0); @@ -449,30 +448,30 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0); * @zq: pointer to zcrypt_queue structure that identifies the * CEXxA device to the request distributor * @mex: pointer to the modexpo request buffer + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. */ static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq, struct ica_rsa_modexpo *mex, struct ap_message *ap_msg) { - struct completion work; int rc; - ap_msg->bufsize = MSGTYPE50_CRB3_MAX_MSG_SIZE; - ap_msg->msg = kmalloc(ap_msg->bufsize, GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; + if (ap_msg->bufsize < MSGTYPE50_CRB3_MAX_MSG_SIZE) + return -EMSGSIZE; ap_msg->receive = zcrypt_msgtype50_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = &work; rc = ICAMEX_msg_to_type50MEX_msg(zq, ap_msg, mex); if (rc) goto out; - init_completion(&work); + init_completion(&ap_msg->response.work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out; - rc = wait_for_completion_interruptible(&work); + rc = wait_for_completion_interruptible(&ap_msg->response.work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -485,11 +484,10 @@ static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq, } out: - ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send me cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("send me cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -499,30 +497,30 @@ out: * @zq: pointer to zcrypt_queue structure that identifies the * CEXxA device to the request distributor * @crt: pointer to the modexpoc_crt request buffer + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. */ static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq, struct ica_rsa_modexpo_crt *crt, struct ap_message *ap_msg) { - struct completion work; int rc; - ap_msg->bufsize = MSGTYPE50_CRB3_MAX_MSG_SIZE; - ap_msg->msg = kmalloc(ap_msg->bufsize, GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; + if (ap_msg->bufsize < MSGTYPE50_CRB3_MAX_MSG_SIZE) + return -EMSGSIZE; ap_msg->receive = zcrypt_msgtype50_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = &work; rc = ICACRT_msg_to_type50CRT_msg(zq, ap_msg, crt); if (rc) goto out; - init_completion(&work); + init_completion(&ap_msg->response.work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out; - rc = wait_for_completion_interruptible(&work); + rc = wait_for_completion_interruptible(&ap_msg->response.work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -535,11 +533,10 @@ static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq, } out: - ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send crt cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("send crt cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 3c53abbdc342..a0dcab5dc4f2 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -10,8 +10,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/module.h> #include <linux/init.h> @@ -31,11 +30,6 @@ #define CEIL4(x) ((((x) + 3) / 4) * 4) -struct response_type { - struct completion work; - int type; -}; - #define CEXXC_RESPONSE_TYPE_ICA 0 #define CEXXC_RESPONSE_TYPE_XCRB 1 #define CEXXC_RESPONSE_TYPE_EP11 2 @@ -437,9 +431,8 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, ap_msg->flags |= AP_MSG_FLAG_ADMIN; break; default: - ZCRYPT_DBF_DBG("%s unknown CPRB minor version '%c%c'\n", - __func__, msg->cprbx.func_id[0], - msg->cprbx.func_id[1]); + pr_debug("unknown CPRB minor version '%c%c'\n", + msg->cprbx.func_id[0], msg->cprbx.func_id[1]); } /* copy data block */ @@ -629,9 +622,8 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy CPRB to user */ if (xcrb->reply_control_blk_length < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_control_blk_length, - msg->fmt2.count1); + pr_debug("reply_control_blk_length %u < required %u => EMSGSIZE\n", + xcrb->reply_control_blk_length, msg->fmt2.count1); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_control_blk_addr, @@ -642,9 +634,8 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy data buffer to user */ if (msg->fmt2.count2) { if (xcrb->reply_data_length < msg->fmt2.count2) { - ZCRYPT_DBF_DBG("%s reply_data_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_data_length, - msg->fmt2.count2); + pr_debug("reply_data_length %u < required %u => EMSGSIZE\n", + xcrb->reply_data_length, msg->fmt2.count2); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_data_addr, @@ -673,9 +664,8 @@ static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, char *data = reply->msg; if (xcrb->resp_len < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s resp_len %u < required %u => EMSGSIZE\n", - __func__, (unsigned int)xcrb->resp_len, - msg->fmt2.count1); + pr_debug("resp_len %u < required %u => EMSGSIZE\n", + (unsigned int)xcrb->resp_len, msg->fmt2.count1); return -EMSGSIZE; } @@ -860,7 +850,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, .type = TYPE82_RSP_CODE, .reply_code = REP82_ERROR_MACHINE_FAILURE, }; - struct response_type *resp_type = msg->private; + struct ap_response_type *resp_type = &msg->response; struct type86x_reply *t86r; int len; @@ -875,7 +865,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = sizeof(struct type86x_reply) + t86r->length; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -889,7 +879,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -924,7 +914,7 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, .type = TYPE82_RSP_CODE, .reply_code = REP82_ERROR_MACHINE_FAILURE, }; - struct response_type *resp_type = msg->private; + struct ap_response_type *resp_type = &msg->response; struct type86_ep11_reply *t86r; int len; @@ -939,7 +929,7 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -971,9 +961,7 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, struct ica_rsa_modexpo *mex, struct ap_message *ap_msg) { - struct response_type resp_type = { - .type = CEXXC_RESPONSE_TYPE_ICA, - }; + struct ap_response_type *resp_type = &ap_msg->response; int rc; ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); @@ -983,15 +971,15 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = &resp_type; rc = icamex_msg_to_type6mex_msgx(zq, ap_msg, mex); if (rc) goto out_free; - init_completion(&resp_type.work); + resp_type->type = CEXXC_RESPONSE_TYPE_ICA; + init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out_free; - rc = wait_for_completion_interruptible(&resp_type.work); + rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -1005,7 +993,6 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, out_free: free_page((unsigned long)ap_msg->msg); - ap_msg->private = NULL; ap_msg->msg = NULL; return rc; } @@ -1021,9 +1008,7 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, struct ica_rsa_modexpo_crt *crt, struct ap_message *ap_msg) { - struct response_type resp_type = { - .type = CEXXC_RESPONSE_TYPE_ICA, - }; + struct ap_response_type *resp_type = &ap_msg->response; int rc; ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); @@ -1033,15 +1018,15 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = &resp_type; rc = icacrt_msg_to_type6crt_msgx(zq, ap_msg, crt); if (rc) goto out_free; - init_completion(&resp_type.work); + resp_type->type = CEXXC_RESPONSE_TYPE_ICA; + init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out_free; - rc = wait_for_completion_interruptible(&resp_type.work); + rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -1055,7 +1040,6 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, out_free: free_page((unsigned long)ap_msg->msg); - ap_msg->private = NULL; ap_msg->msg = NULL; return rc; } @@ -1065,28 +1049,21 @@ out_free: * Prepare a CCA AP msg: fetch the required data from userspace, * prepare the AP msg, fill some info into the ap_message struct, * extract some data from the CPRB and give back to the caller. - * This function allocates memory and needs an ap_msg prepared - * by the caller with ap_init_message(). Also the caller has to - * make sure ap_release_message() is always called even on failure. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. */ int prep_cca_ap_msg(bool userspace, struct ica_xcRB *xcrb, struct ap_message *ap_msg, unsigned int *func_code, unsigned short **dom) { - struct response_type resp_type = { - .type = CEXXC_RESPONSE_TYPE_XCRB, - }; + struct ap_response_type *resp_type = &ap_msg->response; - ap_msg->bufsize = atomic_read(&ap_max_msg_size); - ap_msg->msg = kmalloc(ap_msg->bufsize, GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = kmemdup(&resp_type, sizeof(resp_type), GFP_KERNEL); - if (!ap_msg->private) - return -ENOMEM; + resp_type->type = CEXXC_RESPONSE_TYPE_XCRB; return xcrb_msg_to_type6cprb_msgx(userspace, ap_msg, xcrb, func_code, dom); } @@ -1101,7 +1078,7 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, struct ica_xcRB *xcrb, struct ap_message *ap_msg) { - struct response_type *rtype = ap_msg->private; + struct ap_response_type *resp_type = &ap_msg->response; struct { struct type6_hdr hdr; struct CPRBX cprbx; @@ -1132,11 +1109,11 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, msg->hdr.fromcardlen1 -= delta; } - init_completion(&rtype->work); + init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out; - rc = wait_for_completion_interruptible(&rtype->work); + rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -1151,9 +1128,9 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("send cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -1162,28 +1139,21 @@ out: * Prepare an EP11 AP msg: fetch the required data from userspace, * prepare the AP msg, fill some info into the ap_message struct, * extract some data from the CPRB and give back to the caller. - * This function allocates memory and needs an ap_msg prepared - * by the caller with ap_init_message(). Also the caller has to - * make sure ap_release_message() is always called even on failure. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. */ int prep_ep11_ap_msg(bool userspace, struct ep11_urb *xcrb, struct ap_message *ap_msg, unsigned int *func_code, unsigned int *domain) { - struct response_type resp_type = { - .type = CEXXC_RESPONSE_TYPE_EP11, - }; + struct ap_response_type *resp_type = &ap_msg->response; - ap_msg->bufsize = atomic_read(&ap_max_msg_size); - ap_msg->msg = kmalloc(ap_msg->bufsize, GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; ap_msg->receive = zcrypt_msgtype6_receive_ep11; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = kmemdup(&resp_type, sizeof(resp_type), GFP_KERNEL); - if (!ap_msg->private) - return -ENOMEM; + resp_type->type = CEXXC_RESPONSE_TYPE_EP11; return xcrb_msg_to_type6_ep11cprb_msgx(userspace, ap_msg, xcrb, func_code, domain); } @@ -1201,7 +1171,7 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * { int rc; unsigned int lfmt; - struct response_type *rtype = ap_msg->private; + struct ap_response_type *resp_type = &ap_msg->response; struct { struct type6_hdr hdr; struct ep11_cprb cprbx; @@ -1255,11 +1225,11 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * msg->hdr.fromcardlen1 = zq->reply.bufsize - sizeof(struct type86_hdr) - sizeof(struct type86_fmt2_ext); - init_completion(&rtype->work); + init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out; - rc = wait_for_completion_interruptible(&rtype->work); + rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) @@ -1274,29 +1244,31 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("send cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } +/* + * Prepare a CEXXC get random request ap message. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_max_msg_size is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. + */ int prep_rng_ap_msg(struct ap_message *ap_msg, int *func_code, unsigned int *domain) { - struct response_type resp_type = { - .type = CEXXC_RESPONSE_TYPE_XCRB, - }; + struct ap_response_type *resp_type = &ap_msg->response; - ap_msg->bufsize = AP_DEFAULT_MAX_MSG_SIZE; - ap_msg->msg = kmalloc(ap_msg->bufsize, GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; + if (ap_msg->bufsize < AP_DEFAULT_MAX_MSG_SIZE) + return -EMSGSIZE; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); - ap_msg->private = kmemdup(&resp_type, sizeof(resp_type), GFP_KERNEL); - if (!ap_msg->private) - return -ENOMEM; + + resp_type->type = CEXXC_RESPONSE_TYPE_XCRB; rng_type6cprb_msgx(ap_msg, ZCRYPT_RNG_BUFFER_SIZE, domain); @@ -1323,16 +1295,16 @@ static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq, short int verb_length; short int key_length; } __packed * msg = ap_msg->msg; - struct response_type *rtype = ap_msg->private; + struct ap_response_type *resp_type = &ap_msg->response; int rc; msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid); - init_completion(&rtype->work); + init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) goto out; - rc = wait_for_completion_interruptible(&rtype->work); + rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c index 112a80e8e6c2..a173d32eb6e8 100644 --- a/drivers/s390/crypto/zcrypt_queue.c +++ b/drivers/s390/crypto/zcrypt_queue.c @@ -11,6 +11,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ +#include <linux/export.h> #include <linux/module.h> #include <linux/init.h> #include <linux/interrupt.h> @@ -18,7 +19,6 @@ #include <linux/fs.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> @@ -42,7 +42,7 @@ static ssize_t online_show(struct device *dev, { struct zcrypt_queue *zq = dev_get_drvdata(dev); struct ap_queue *aq = to_ap_queue(dev); - int online = aq->config && zq->online ? 1 : 0; + int online = aq->config && !aq->chkstop && zq->online ? 1 : 0; return sysfs_emit(buf, "%d\n", online); } @@ -59,7 +59,8 @@ static ssize_t online_store(struct device *dev, if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1) return -EINVAL; - if (online && (!aq->config || !aq->card->config)) + if (online && (!aq->config || !aq->card->config || + aq->chkstop || aq->card->chkstop)) return -ENODEV; if (online && !zc->online) return -EINVAL; |