diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c | 755 |
1 files changed, 728 insertions, 27 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index e995f8378df7..3a94b8f8031e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -2,53 +2,253 @@ // Copyright (c) 2019 Mellanox Technologies. #include "mlx5_core.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" -int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, - u32 key_type, u32 *p_key_id) -{ - u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; - u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; - u32 sz_bits = sz_bytes * BITS_PER_BYTE; - u8 general_obj_key_size; - u64 general_obj_types; - void *obj, *key_p; - int err; +#define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) +#define type2idx(type) ((type) - 1) - obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); - key_p = MLX5_ADDR_OF(encryption_key_obj, obj, key); +#define MLX5_CRYPTO_DEK_POOL_SYNC_THRESH 128 - general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); - if (!(general_obj_types & - MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) - return -EINVAL; +/* calculate the num of DEKs, which are freed by any user + * (for example, TLS) after last revalidation in a pool or a bulk. + */ +#define MLX5_CRYPTO_DEK_CALC_FREED(a) \ + ({ typeof(a) _a = (a); \ + _a->num_deks - _a->avail_deks - _a->in_use_deks; }) + +#define MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) MLX5_CRYPTO_DEK_CALC_FREED(pool) +#define MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk) MLX5_CRYPTO_DEK_CALC_FREED(bulk) + +#define MLX5_CRYPTO_DEK_BULK_IDLE(bulk) \ + ({ typeof(bulk) _bulk = (bulk); \ + _bulk->avail_deks == _bulk->num_deks; }) + +enum { + MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0), +}; + +struct mlx5_crypto_dek_pool { + struct mlx5_core_dev *mdev; + u32 key_purpose; + int num_deks; /* the total number of keys in this pool */ + int avail_deks; /* the number of available keys in this pool */ + int in_use_deks; /* the number of being used keys in this pool */ + struct mutex lock; /* protect the following lists, and the bulks */ + struct list_head partial_list; /* some of keys are available */ + struct list_head full_list; /* no available keys */ + struct list_head avail_list; /* all keys are available to use */ + + /* No in-used keys, and all need to be synced. + * These bulks will be put to avail list after sync. + */ + struct list_head sync_list; + + bool syncing; + struct list_head wait_for_free; + struct work_struct sync_work; + + spinlock_t destroy_lock; /* protect destroy_list */ + struct list_head destroy_list; + struct work_struct destroy_work; +}; + +struct mlx5_crypto_dek_bulk { + struct mlx5_core_dev *mdev; + int base_obj_id; + int avail_start; /* the bit to start search */ + int num_deks; /* the total number of keys in a bulk */ + int avail_deks; /* the number of keys available, with need_sync bit 0 */ + int in_use_deks; /* the number of keys being used, with in_use bit 1 */ + struct list_head entry; + + /* 0: not being used by any user, 1: otherwise */ + unsigned long *in_use; + + /* The bits are set when they are used, and reset after crypto_sync + * is executed. So, the value 0 means the key is newly created, or not + * used after sync, and 1 means it is in use, or freed but not synced + */ + unsigned long *need_sync; +}; + +struct mlx5_crypto_dek_priv { + struct mlx5_core_dev *mdev; + int log_dek_obj_range; +}; + +struct mlx5_crypto_dek { + struct mlx5_crypto_dek_bulk *bulk; + struct list_head entry; + u32 obj_id; +}; + +u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek) +{ + return dek->obj_id; +} + +static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev, + u32 sz_bytes, u8 *key_sz_p) +{ + u32 sz_bits = sz_bytes * BITS_PER_BYTE; switch (sz_bits) { case 128: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; - key_p += sz_bytes; + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; break; case 256: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; break; default: + mlx5_core_err(mdev, "Crypto offload error, invalid key size (%u bits)\n", + sz_bits); return -EINVAL; } - memcpy(key_p, key, sz_bytes); + return 0; +} + +static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, + const void *key, u32 sz_bytes) +{ + void *dst; + u8 key_sz; + int err; + + err = mlx5_crypto_dek_get_key_sz(mdev, sz_bytes, &key_sz); + if (err) + return err; + + MLX5_SET(encryption_key_obj, key_obj, key_size, key_sz); + + if (sz_bytes == 16) + /* For key size of 128b the MSBs are reserved. */ + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key[1]); + else + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key); + + memcpy(dst, key, sz_bytes); + + return 0; +} + +static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev, + int crypto_type) +{ + u32 in[MLX5_ST_SZ_DW(sync_crypto_in)] = {}; + int err; + + mlx5_core_dbg(mdev, + "Execute SYNC_CRYPTO command with crypto_type(0x%x)\n", + crypto_type); + + MLX5_SET(sync_crypto_in, in, opcode, MLX5_CMD_OP_SYNC_CRYPTO); + MLX5_SET(sync_crypto_in, in, crypto_type, crypto_type); + + err = mlx5_cmd_exec_in(mdev, sync_crypto, in); + if (err) + mlx5_core_err(mdev, + "Failed to exec sync crypto, type=%d, err=%d\n", + crypto_type, err); + + return err; +} + +static int mlx5_crypto_create_dek_bulk(struct mlx5_core_dev *mdev, + u32 key_purpose, int log_obj_range, + u32 *obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; - MLX5_SET(encryption_key_obj, obj, key_size, general_obj_key_size); - MLX5_SET(encryption_key_obj, obj, key_type, key_type); MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_create_param, param, log_obj_range, log_obj_range); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + mlx5_core_dbg(mdev, "DEK objects created, bulk=%d, obj_id=%d\n", + 1 << log_obj_range, *obj_id); + + return 0; +} + +static int mlx5_crypto_modify_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, u32 key_purpose, + u32 obj_id, u32 obj_offset) +{ + u32 in[MLX5_ST_SZ_DW(modify_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); + + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_query_param, param, obj_offset, obj_offset); + + obj = MLX5_ADDR_OF(modify_encryption_key_in, in, encryption_key_object); + MLX5_SET64(encryption_key_obj, obj, modify_field_select, 1); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + + /* avoid leaking key on the stack */ + memzero_explicit(in, sizeof(in)); + + return err; +} + +static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_purpose, u32 *p_key_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + u64 general_obj_types; + void *obj; + int err; + + general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); + if (!(general_obj_types & + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) + return -EINVAL; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) *p_key_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); @@ -58,7 +258,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, return err; } -void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +static void mlx5_crypto_destroy_dek_key(struct mlx5_core_dev *mdev, u32 key_id) { u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; @@ -71,3 +271,504 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); } + +int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id) +{ + return mlx5_crypto_create_dek_key(mdev, key, sz_bytes, key_type, p_key_id); +} + +void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +{ + mlx5_crypto_destroy_dek_key(mdev, key_id); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_bulk_create(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_priv *dek_priv = pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = pool->mdev; + struct mlx5_crypto_dek_bulk *bulk; + int num_deks, base_obj_id; + int err; + + bulk = kzalloc(sizeof(*bulk), GFP_KERNEL); + if (!bulk) + return ERR_PTR(-ENOMEM); + + num_deks = 1 << dek_priv->log_dek_obj_range; + bulk->need_sync = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->need_sync) { + err = -ENOMEM; + goto err_out; + } + + bulk->in_use = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->in_use) { + err = -ENOMEM; + goto err_out; + } + + err = mlx5_crypto_create_dek_bulk(mdev, pool->key_purpose, + dek_priv->log_dek_obj_range, + &base_obj_id); + if (err) + goto err_out; + + bulk->base_obj_id = base_obj_id; + bulk->num_deks = num_deks; + bulk->avail_deks = num_deks; + bulk->mdev = mdev; + + return bulk; + +err_out: + bitmap_free(bulk->in_use); + bitmap_free(bulk->need_sync); + kfree(bulk); + return ERR_PTR(err); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_add_bulk(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk; + + bulk = mlx5_crypto_dek_bulk_create(pool); + if (IS_ERR(bulk)) + return bulk; + + pool->avail_deks += bulk->num_deks; + pool->num_deks += bulk->num_deks; + list_add(&bulk->entry, &pool->partial_list); + + return bulk; +} + +static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk) +{ + mlx5_crypto_destroy_dek_key(bulk->mdev, bulk->base_obj_id); + bitmap_free(bulk->need_sync); + bitmap_free(bulk->in_use); + kfree(bulk); +} + +static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk, + bool delay) +{ + pool->num_deks -= bulk->num_deks; + pool->avail_deks -= bulk->avail_deks; + pool->in_use_deks -= bulk->in_use_deks; + list_del(&bulk->entry); + if (!delay) + mlx5_crypto_dek_bulk_free(bulk); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) +{ + struct mlx5_crypto_dek_bulk *bulk; + int pos; + + mutex_lock(&pool->lock); + bulk = list_first_entry_or_null(&pool->partial_list, + struct mlx5_crypto_dek_bulk, entry); + + if (bulk) { + pos = find_next_zero_bit(bulk->need_sync, bulk->num_deks, + bulk->avail_start); + if (pos == bulk->num_deks) { + mlx5_core_err(pool->mdev, "Wrong DEK bulk avail_start.\n"); + pos = find_first_zero_bit(bulk->need_sync, bulk->num_deks); + } + WARN_ON(pos == bulk->num_deks); + } else { + bulk = list_first_entry_or_null(&pool->avail_list, + struct mlx5_crypto_dek_bulk, + entry); + if (bulk) { + list_move(&bulk->entry, &pool->partial_list); + } else { + bulk = mlx5_crypto_dek_pool_add_bulk(pool); + if (IS_ERR(bulk)) + goto out; + } + pos = 0; + } + + *obj_offset = pos; + bitmap_set(bulk->need_sync, pos, 1); + bitmap_set(bulk->in_use, pos, 1); + bulk->in_use_deks++; + bulk->avail_deks--; + if (!bulk->avail_deks) { + list_move(&bulk->entry, &pool->full_list); + bulk->avail_start = bulk->num_deks; + } else { + bulk->avail_start = pos + 1; + } + pool->avail_deks--; + pool->in_use_deks++; + +out: + mutex_unlock(&pool->lock); + return bulk; +} + +static bool mlx5_crypto_dek_need_sync(struct mlx5_crypto_dek_pool *pool) +{ + return !pool->syncing && + MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) > MLX5_CRYPTO_DEK_POOL_SYNC_THRESH; +} + +static int mlx5_crypto_dek_free_locked(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_crypto_dek_bulk *bulk = dek->bulk; + int obj_offset; + bool old_val; + int err = 0; + + obj_offset = dek->obj_id - bulk->base_obj_id; + old_val = test_and_clear_bit(obj_offset, bulk->in_use); + WARN_ON_ONCE(!old_val); + if (!old_val) { + err = -ENOENT; + goto out_free; + } + pool->in_use_deks--; + bulk->in_use_deks--; + if (!bulk->avail_deks && !bulk->in_use_deks) + list_move(&bulk->entry, &pool->sync_list); + + if (mlx5_crypto_dek_need_sync(pool) && schedule_work(&pool->sync_work)) + pool->syncing = true; + +out_free: + kfree(dek); + return err; +} + +static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + int err = 0; + + mutex_lock(&pool->lock); + if (pool->syncing) + list_add(&dek->entry, &pool->wait_for_free); + else + err = mlx5_crypto_dek_free_locked(pool, dek); + mutex_unlock(&pool->lock); + + return err; +} + +/* Update the bits for a bulk while sync, and avail_next for search. + * As the combinations of (need_sync, in_use) of one DEK are + * - (0,0) means the key is ready for use, + * - (1,1) means the key is currently being used by a user, + * - (1,0) means the key is freed, and waiting for being synced, + * - (0,1) is invalid state. + * the number of revalidated DEKs can be calculated by + * hweight_long(need_sync XOR in_use), and the need_sync bits can be reset + * by simply copying from in_use bits. + */ +static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk) +{ + unsigned long *need_sync = bulk->need_sync; + unsigned long *in_use = bulk->in_use; + int i, freed, reused, avail_next; + bool first = true; + + freed = MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk); + + for (i = 0; freed && i < BITS_TO_LONGS(bulk->num_deks); + i++, need_sync++, in_use++) { + reused = hweight_long((*need_sync) ^ (*in_use)); + if (!reused) + continue; + + bulk->avail_deks += reused; + pool->avail_deks += reused; + *need_sync = *in_use; + if (first) { + avail_next = i * BITS_PER_TYPE(long); + if (bulk->avail_start > avail_next) + bulk->avail_start = avail_next; + first = false; + } + + freed -= reused; + } +} + +/* Return true if the bulk is reused, false if destroyed with delay */ +static bool mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk, + struct list_head *destroy_list) +{ + if (list_empty(&pool->avail_list)) { + list_move(&bulk->entry, &pool->avail_list); + return true; + } + + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true); + list_add(&bulk->entry, destroy_list); + return false; +} + +static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool, + struct list_head *list, + struct list_head *head) +{ + spin_lock(&pool->destroy_lock); + list_splice_init(list, head); + spin_unlock(&pool->destroy_lock); +} + +static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek *dek, *next; + + list_for_each_entry_safe(dek, next, &pool->wait_for_free, entry) { + list_del(&dek->entry); + mlx5_crypto_dek_free_locked(pool, dek); + } +} + +/* For all the bulks in each list, reset the bits while sync. + * Move them to different lists according to the number of available DEKs. + * Destrory all the idle bulks, except one for quick service. + * And free DEKs in the waiting list at the end of this func. + */ +static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + LIST_HEAD(destroy_list); + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + + if (!bulk->avail_deks) + continue; + + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); + else + list_move(&bulk->entry, &pool->partial_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) { + bulk->avail_deks = bulk->num_deks; + pool->avail_deks += bulk->num_deks; + if (mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list)) { + memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); + bulk->avail_start = 0; + } + } + + mlx5_crypto_dek_pool_free_wait_keys(pool); + + if (!list_empty(&destroy_list)) { + mlx5_crypto_dek_pool_splice_destroy_list(pool, &destroy_list, + &pool->destroy_list); + schedule_work(&pool->destroy_work); + } +} + +static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, sync_work); + int err; + + err = mlx5_crypto_cmd_sync_crypto(pool->mdev, BIT(pool->key_purpose)); + mutex_lock(&pool->lock); + if (!err) + mlx5_crypto_dek_pool_reset_synced(pool); + pool->syncing = false; + mutex_unlock(&pool->lock); +} + +struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, + const void *key, u32 sz_bytes) +{ + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = dek_pool->mdev; + u32 key_purpose = dek_pool->key_purpose; + struct mlx5_crypto_dek_bulk *bulk; + struct mlx5_crypto_dek *dek; + int obj_offset; + int err; + + dek = kzalloc(sizeof(*dek), GFP_KERNEL); + if (!dek) + return ERR_PTR(-ENOMEM); + + if (!dek_priv) { + err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, + key_purpose, &dek->obj_id); + goto out; + } + + bulk = mlx5_crypto_dek_pool_pop(dek_pool, &obj_offset); + if (IS_ERR(bulk)) { + err = PTR_ERR(bulk); + goto out; + } + + dek->bulk = bulk; + dek->obj_id = bulk->base_obj_id + obj_offset; + err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose, + bulk->base_obj_id, obj_offset); + if (err) { + mlx5_crypto_dek_pool_push(dek_pool, dek); + return ERR_PTR(err); + } + +out: + if (err) { + kfree(dek); + return ERR_PTR(err); + } + + return dek; +} + +void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = dek_pool->mdev; + + if (!dek_priv) { + mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + kfree(dek); + } else { + mlx5_crypto_dek_pool_push(dek_pool, dek); + } +} + +static void mlx5_crypto_dek_free_destroy_list(struct list_head *destroy_list) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + list_for_each_entry_safe(bulk, tmp, destroy_list, entry) + mlx5_crypto_dek_bulk_free(bulk); +} + +static void mlx5_crypto_dek_destroy_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, destroy_work); + LIST_HEAD(destroy_list); + + mlx5_crypto_dek_pool_splice_destroy_list(pool, &pool->destroy_list, + &destroy_list); + mlx5_crypto_dek_free_destroy_list(&destroy_list); +} + +struct mlx5_crypto_dek_pool * +mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) +{ + struct mlx5_crypto_dek_pool *pool; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return ERR_PTR(-ENOMEM); + + pool->mdev = mdev; + pool->key_purpose = key_purpose; + + mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->avail_list); + INIT_LIST_HEAD(&pool->partial_list); + INIT_LIST_HEAD(&pool->full_list); + INIT_LIST_HEAD(&pool->sync_list); + INIT_LIST_HEAD(&pool->wait_for_free); + INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn); + spin_lock_init(&pool->destroy_lock); + INIT_LIST_HEAD(&pool->destroy_list); + INIT_WORK(&pool->destroy_work, mlx5_crypto_dek_destroy_work_fn); + + return pool; +} + +void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + cancel_work_sync(&pool->sync_work); + cancel_work_sync(&pool->destroy_work); + + mlx5_crypto_dek_pool_free_wait_keys(pool); + + list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + mlx5_crypto_dek_free_destroy_list(&pool->destroy_list); + + mutex_destroy(&pool->lock); + + kfree(pool); +} + +void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) +{ + if (!dek_priv) + return; + + kfree(dek_priv); +} + +struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_crypto_dek_priv *dek_priv; + int err; + + if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)) + return NULL; + + dek_priv = kzalloc(sizeof(*dek_priv), GFP_KERNEL); + if (!dek_priv) + return ERR_PTR(-ENOMEM); + + dek_priv->mdev = mdev; + dek_priv->log_dek_obj_range = min_t(int, 12, + MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)); + + /* sync all types of objects */ + err = mlx5_crypto_cmd_sync_crypto(mdev, MLX5_CRYPTO_DEK_ALL_TYPE); + if (err) + goto err_sync_crypto; + + mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n", + 1 << dek_priv->log_dek_obj_range, + 1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc), + 1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks)); + + return dek_priv; + +err_sync_crypto: + kfree(dek_priv); + return ERR_PTR(err); +} |