Synchronize internal development codes to keep basic functional codes consistent.
Signed-off-by: Longfang Liu liulongfang@huawei.com --- drv/hash_mb/hash_mb.c | 83 +++++----- drv/hisi_comp.c | 45 ++--- drv/hisi_dae.c | 32 ++-- drv/hisi_hpre.c | 358 +++++++++++++++++++--------------------- drv/hisi_qm_udrv.c | 93 +++++------ drv/hisi_qm_udrv.h | 11 +- drv/hisi_sec.c | 150 +++++++++-------- drv/isa_ce_sm3.c | 37 ++--- drv/isa_ce_sm4.c | 41 ++--- include/wd.h | 21 +-- include/wd_aead.h | 1 + include/wd_alg.h | 34 +--- include/wd_alg_common.h | 28 ++-- include/wd_comp.h | 3 +- include/wd_digest.h | 9 +- include/wd_util.h | 27 +-- wd.c | 12 +- wd_alg.c | 2 +- wd_cipher.c | 50 ++++-- wd_comp.c | 34 ++-- wd_digest.c | 41 ++--- wd_sched.c | 3 +- wd_util.c | 140 ++++++++++------ 23 files changed, 625 insertions(+), 630 deletions(-)
diff --git a/drv/hash_mb/hash_mb.c b/drv/hash_mb/hash_mb.c index e4a9564..3f5b1a5 100644 --- a/drv/hash_mb/hash_mb.c +++ b/drv/hash_mb/hash_mb.c @@ -1,5 +1,15 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ +/* + * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. + */
#include <sys/auxv.h> #include <pthread.h> @@ -19,7 +29,6 @@ #define HASH_PADDING_BLOCKS 2 #define HASH_NENO_PROCESS_JOBS 4 #define HASH_TRY_PROCESS_COUNT 16 -#define BYTES_TO_BITS_OFFSET 3
#define MD5_DIGEST_DATA_SIZE 16 #define SM3_DIGEST_DATA_SIZE 32 @@ -109,7 +118,7 @@ static void hash_mb_queue_uninit(struct wd_ctx_config_internal *config, int ctx_ int i;
for (i = 0; i < ctx_num; i++) { - ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; + ctx = (struct wd_soft_ctx *)(uintptr_t)config->ctxs[i].ctx; mb_queue = ctx->priv; pthread_spin_destroy(&mb_queue->r_lock); hash_mb_uninit_poll_queue(&mb_queue->sm3_poll_queue); @@ -150,7 +159,7 @@ static int hash_mb_queue_init(struct wd_ctx_config_internal *config) }
mb_queue->ctx_mode = config->ctxs[i].ctx_mode; - ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; + ctx = (struct wd_soft_ctx *)(uintptr_t)config->ctxs[i].ctx; ctx->priv = mb_queue; ret = hash_mb_init_poll_queue(&mb_queue->sm3_poll_queue); if (ret) @@ -186,52 +195,41 @@ free_mb_queue: return ret; }
-static int hash_mb_init(struct wd_alg_driver *drv, void *conf) +static int hash_mb_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct hash_mb_ctx *priv; - int ret; + struct hash_mb_ctx *mb_ctx = priv;
/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;
- priv = malloc(sizeof(struct hash_mb_ctx)); - if (!priv) - return -WD_ENOMEM; - /* multibuff does not use epoll. */ config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - - ret = hash_mb_queue_init(config); - if (ret) { - free(priv); - return ret; - } + memcpy(&mb_ctx->config, config, sizeof(struct wd_ctx_config_internal));
- drv->priv = priv; - - return WD_SUCCESS; + return hash_mb_queue_init(config); }
-static void hash_mb_exit(struct wd_alg_driver *drv) +static void hash_mb_exit(void *priv) { - struct hash_mb_ctx *priv = (struct hash_mb_ctx *)drv->priv; + struct hash_mb_ctx *mb_ctx = priv; + struct wd_ctx_config_internal *config;
- if (!priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + }
- hash_mb_queue_uninit(&priv->config, priv->config.ctx_num); - free(priv); - drv->priv = NULL; + config = &mb_ctx->config; + hash_mb_queue_uninit(config, config->ctx_num); }
static void hash_mb_pad_data(struct hash_pad *hash_pad, __u8 *in, __u32 partial, __u64 total_len, bool transfer) { - __u64 size = total_len << BYTES_TO_BITS_OFFSET; __u8 *buffer = hash_pad->pad; + __u64 size = total_len << 3;
if (partial) memcpy(buffer, in, partial); @@ -266,7 +264,7 @@ static inline void hash_xor(__u8 *key_out, __u8 *key_in, __u32 key_len, __u8 xor if (i < key_len) key_out[i] = key_in[i] ^ xor_value; else - key_out[i] = xor_value; + key_out[i] = 0x0 ^ xor_value; } }
@@ -391,8 +389,7 @@ static int hash_first_block_process(struct wd_digest_msg *d_msg, memcpy(d_msg->partial_block, buffer, d_msg->partial_bytes); }
- /* - * Long hash mode, if first block is less than HASH_BLOCK_SIZE, + /* Long hash mode, if first block is less than HASH_BLOCK_SIZE, * copy ikey hash result to out. */ if (!job->len) { @@ -550,9 +547,9 @@ static int hash_mb_check_param(struct hash_mb_queue *mb_queue, struct wd_digest_ return WD_SUCCESS; }
-static int hash_mb_send(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static int hash_mb_send(handle_t ctx, void *drv_msg) { - struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; + struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)(uintptr_t)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; struct wd_digest_msg *d_msg = drv_msg; struct hash_mb_poll_queue *poll_queue; @@ -771,9 +768,9 @@ static int hash_mb_do_jobs(struct hash_mb_queue *mb_queue) return WD_SUCCESS; }
-static int hash_mb_recv(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static int hash_mb_recv(handle_t ctx, void *drv_msg) { - struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; + struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)(uintptr_t)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; struct wd_digest_msg *msg = drv_msg; int ret, i = 0; @@ -805,6 +802,7 @@ static int hash_mb_get_usage(void *param) .alg_name = (hash_alg_name),\ .calc_type = UADK_ALG_SVE_INSTR,\ .priority = 100,\ + .priv_size = sizeof(struct hash_mb_ctx),\ .queue_num = 1,\ .op_type_num = 1,\ .fallback = 0,\ @@ -822,14 +820,18 @@ static struct wd_alg_driver hash_mb_driver[] = {
static void __attribute__((constructor)) hash_mb_probe(void) { + unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); size_t i; int ret;
+ if (!(auxval & HWCAP_SVE)) + return; + WD_INFO("Info: register hash_mb alg drivers!\n"); for (i = 0; i < alg_num; i++) { ret = wd_alg_driver_register(&hash_mb_driver[i]); - if (ret && ret != -WD_ENODEV) + if (ret) WD_ERR("Error: register hash multibuff %s failed!\n", hash_mb_driver[i].alg_name); } @@ -837,11 +839,14 @@ static void __attribute__((constructor)) hash_mb_probe(void)
static void __attribute__((destructor)) hash_mb_remove(void) { + unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); size_t i;
+ if (!(auxval & HWCAP_SVE)) + return; + WD_INFO("Info: unregister hash_mb alg drivers!\n"); for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&hash_mb_driver[i]); } - diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 4c8e18b..ffe8eac 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -338,8 +338,7 @@ static void fill_buf_sgl_skip(struct hisi_zip_sqe *sqe, __u32 src_skip, }
static int fill_buf_deflate_slg_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg, const char *head, - int head_size) + struct wd_comp_msg *msg, const char *head, int head_size) { struct wd_comp_req *req = &msg->req; __u32 out_size = msg->avail_out; @@ -510,6 +509,8 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_type_sgl(sqe);
seq_start = get_seq_start_list(req); + if (unlikely(!seq_start)) + return -WD_EINVAL;
data->literals_start = req->list_dst; data->sequences_start = seq_start; @@ -700,7 +701,7 @@ static void get_data_size_lz77_zstd(struct hisi_zip_sqe *sqe, enum wd_comp_op_ty struct wd_lz77_zstd_data *data = recv_msg->req.priv; void *ctx_buf = recv_msg->ctx_buf;
- if (unlikely(!data)) + if (!data) return;
data->lit_num = sqe->comp_data_length; @@ -785,11 +786,11 @@ static void hisi_zip_sqe_ops_adapt(handle_t h_qp) } }
-static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) +static int hisi_zip_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_zip_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -799,11 +800,7 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }
- priv = malloc(sizeof(struct hisi_zip_ctx)); - if (!priv) - return -WD_EINVAL; - - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); + memcpy(&zip_ctx->config, config, sizeof(struct wd_ctx_config_internal)); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { h_ctx = config->ctxs[i].ctx; @@ -821,7 +818,6 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) }
hisi_zip_sqe_ops_adapt(h_qp); - drv->priv = priv;
return 0; out: @@ -829,29 +825,20 @@ out: h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); hisi_qm_free_qp(h_qp); } - free(priv); return -WD_EINVAL; }
-static void hisi_zip_exit(struct wd_alg_driver *drv) +static void hisi_zip_exit(void *priv) { - struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; - struct wd_ctx_config_internal *config; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; + struct wd_ctx_config_internal *config = &zip_ctx->config; handle_t h_qp; __u32 i;
- if (!priv) { - /* return if already exit */ - return; - } - - config = &priv->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; }
static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, @@ -931,7 +918,7 @@ static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, } }
-static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_send(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *msg = comp_msg; @@ -1001,14 +988,13 @@ static void get_ctx_buf(struct hisi_zip_sqe *sqe, }
static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg) + struct wd_comp_msg *recv_msg) { __u32 buf_type = (sqe->dw9 & HZ_BUF_TYPE_MASK) >> BUF_TYPE_SHIFT; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; __u32 status = sqe->dw3 & HZ_STATUS_MASK; __u32 type = sqe->dw9 & HZ_REQ_TYPE_MASK; - struct wd_comp_msg *recv_msg = msg; bool need_debug = wd_need_debug(); int alg_type, ret; __u32 tag; @@ -1045,7 +1031,6 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, }
ops[alg_type].get_data_size(sqe, qp->q_info.qc_type, recv_msg); - get_ctx_buf(sqe, recv_msg);
/* last block no space, need resend null size req */ @@ -1054,7 +1039,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe,
if (need_debug) WD_DEBUG("zip recv lst =%hu, ctx_st=0x%x, status=0x%x, alg=%u!\n", - lstblk, ctx_st, status, type); + lstblk, ctx_st, status, type); if (lstblk && (status == HZ_DECOMP_END)) recv_msg->req.status = WD_STREAM_END;
@@ -1071,7 +1056,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, return 0; }
-static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_recv(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *recv_msg = comp_msg; @@ -1093,6 +1078,7 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com .alg_name = (zip_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_zip_ctx),\ .queue_num = ZIP_CTX_Q_NUM_DEF,\ .op_type_num = 2,\ .fallback = 0,\ @@ -1105,7 +1091,6 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com static struct wd_alg_driver zip_alg_driver[] = { GEN_ZIP_ALG_DRIVER("zlib"), GEN_ZIP_ALG_DRIVER("gzip"), - GEN_ZIP_ALG_DRIVER("deflate"), GEN_ZIP_ALG_DRIVER("lz77_zstd"), }; diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index 09d2387..70d01c9 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -518,7 +518,7 @@ static int check_hashagg_param(struct wd_agg_msg *msg) return WD_SUCCESS; }
-static int hashagg_send(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_send(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -651,7 +651,7 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms } }
-static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_recv(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1546,11 +1546,11 @@ update_table: return ret; }
-static int dae_init(struct wd_alg_driver *drv, void *conf) +static int dae_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_dae_ctx *dae_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_dae_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -1561,10 +1561,6 @@ static int dae_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }
- priv = malloc(sizeof(struct hisi_dae_ctx)); - if (!priv) - return -WD_ENOMEM; - qm_priv.op_type = DAE_HASH_AGG_TYPE; qm_priv.sqe_size = sizeof(struct dae_sqe); /* Allocate qp for each context */ @@ -1585,10 +1581,9 @@ static int dae_init(struct wd_alg_driver *drv, void *conf) if (ret) goto free_h_qp; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&dae_ctx->config, config, sizeof(struct wd_ctx_config_internal));
- return WD_SUCCESS; + return 0;
free_h_qp: hisi_qm_free_qp(h_qp); @@ -1598,29 +1593,27 @@ out: dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - free(priv); return ret; }
-static void dae_exit(struct wd_alg_driver *drv) +static void dae_exit(void *priv) { - struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; + struct hisi_dae_ctx *dae_ctx = priv; struct wd_ctx_config_internal *config; handle_t h_qp; __u32 i;
- if (!priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + }
- config = &priv->config; + config = &dae_ctx->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - - free(priv); - drv->priv = NULL; }
static int dae_get_usage(void *param) @@ -1648,6 +1641,7 @@ static struct wd_alg_driver hashagg_driver = { .alg_name = "hashagg", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_dae_ctx), .queue_num = DAE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 37bb5ee..1f02701 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -1,4 +1,3 @@ - /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */
@@ -40,31 +39,31 @@ #define GEN_PARAMS_SZ(key_size) ((key_size) << 1) #define CRT_PARAM_SZ(key_size) ((key_size) >> 1)
-#define WD_TRANS_FAIL 0 +#define WD_TRANS_FAIL 0
enum hpre_alg_type { - HPRE_ALG_NC_NCRT = 0x0, - HPRE_ALG_NC_CRT = 0x1, - HPRE_ALG_KG_STD = 0x2, - HPRE_ALG_KG_CRT = 0x3, - HPRE_ALG_DH_G2 = 0x4, - HPRE_ALG_DH = 0x5, - HPRE_ALG_PRIME = 0x6, - HPRE_ALG_MOD = 0x7, - HPRE_ALG_MOD_INV = 0x8, - HPRE_ALG_MUL = 0x9, - HPRE_ALG_COPRIME = 0xA, - HPRE_ALG_ECC_CURVE_TEST = 0xB, - HPRE_ALG_ECDH_PLUS = 0xC, - HPRE_ALG_ECDH_MULTIPLY = 0xD, - HPRE_ALG_ECDSA_SIGN = 0xE, - HPRE_ALG_ECDSA_VERF = 0xF, - HPRE_ALG_X_DH_MULTIPLY = 0x10, - HPRE_ALG_SM2_KEY_GEN = 0x11, - HPRE_ALG_SM2_SIGN = 0x12, - HPRE_ALG_SM2_VERF = 0x13, - HPRE_ALG_SM2_ENC = 0x14, - HPRE_ALG_SM2_DEC = 0x15 + HW_ALG_NC_NCRT = 0x0, + HW_ALG_NC_CRT = 0x1, + HW_ALG_KG_STD = 0x2, + HW_ALG_KG_CRT = 0x3, + HW_ALG_DH_G2 = 0x4, + HW_ALG_DH = 0x5, + HW_ALG_PRIME = 0x6, + HW_ALG_MOD = 0x7, + HW_ALG_MOD_INV = 0x8, + HW_ALG_MUL = 0x9, + HW_ALG_COPRIME = 0xA, + HW_ALG_ECC_CURVE_TEST = 0xB, + HW_ALG_ECDH_PLUS = 0xC, + HW_ALG_ECDH_MULTIPLY = 0xD, + HW_ALG_ECDSA_SIGN = 0xE, + HW_ALG_ECDSA_VERF = 0xF, + HW_ALG_X_DH_MULTIPLY = 0x10, + HW_ALG_SM2_KEY_GEN = 0x11, + HW_ALG_SM2_SIGN = 0x12, + HW_ALG_SM2_VERF = 0x13, + HW_ALG_SM2_ENC = 0x14, + HW_ALG_SM2_DEC = 0x15 };
enum hpre_alg_name { @@ -141,7 +140,7 @@ static void dump_hpre_msg(void *msg, int alg) } }
-static bool is_hpre_bin_fmt(char *dst, const char *src, __u32 dsz, __u32 bsz) +static bool is_hpre_bin_fmt(char *dst, const char *src, int dsz, int bsz) { const char *temp = src + dsz; int lens = bsz - dsz; @@ -227,17 +226,11 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey,
wd_rsa_get_crt_prikey_params(prikey, &wd_dq, &wd_dp, &wd_qinv, &wd_q, &wd_p); - ret = crypto_bin_to_hpre_bin(wd_dq->data, (const char *)wd_dq->data, - wd_dq->bsize, wd_dq->dsize, "rsa crt dq"); - if (ret) - return ret; - wd_dq->dsize = wd_dq->bsize; - - ret = crypto_bin_to_hpre_bin(wd_dp->data, (const char *)wd_dp->data, - wd_dp->bsize, wd_dp->dsize, "rsa crt dp"); + ret = crypto_bin_to_hpre_bin(wd_p->data, + (const char *)wd_p->data, wd_p->bsize, wd_p->dsize, "rsa crt p"); if (ret) return ret; - wd_dp->dsize = wd_dp->bsize; + wd_p->dsize = wd_p->bsize;
ret = crypto_bin_to_hpre_bin(wd_q->data, (const char *)wd_q->data, wd_q->bsize, wd_q->dsize, "rsa crt q"); @@ -245,12 +238,6 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey, return ret; wd_q->dsize = wd_q->bsize;
- ret = crypto_bin_to_hpre_bin(wd_p->data, - (const char *)wd_p->data, wd_p->bsize, wd_p->dsize, "rsa crt p"); - if (ret) - return ret; - wd_p->dsize = wd_p->bsize; - ret = crypto_bin_to_hpre_bin(wd_qinv->data, (const char *)wd_qinv->data, wd_qinv->bsize, wd_qinv->dsize, "rsa crt qinv"); @@ -258,6 +245,18 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey, return ret; wd_qinv->dsize = wd_qinv->bsize;
+ ret = crypto_bin_to_hpre_bin(wd_dq->data, (const char *)wd_dq->data, + wd_dq->bsize, wd_dq->dsize, "rsa crt dq"); + if (ret) + return ret; + wd_dq->dsize = wd_dq->bsize; + + ret = crypto_bin_to_hpre_bin(wd_dp->data, (const char *)wd_dp->data, + wd_dp->bsize, wd_dp->dsize, "rsa crt dp"); + if (ret) + return ret; + wd_dp->dsize = wd_dp->bsize; + *data = wd_dq->data;
return WD_SUCCESS; @@ -269,18 +268,18 @@ static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data) int ret;
wd_rsa_get_prikey_params(prikey, &wd_d, &wd_n); - ret = crypto_bin_to_hpre_bin(wd_d->data, (const char *)wd_d->data, - wd_d->bsize, wd_d->dsize, "rsa d"); - if (ret) - return ret; - wd_d->dsize = wd_d->bsize; - ret = crypto_bin_to_hpre_bin(wd_n->data, (const char *)wd_n->data, wd_n->bsize, wd_n->dsize, "rsa n"); if (ret) return ret; wd_n->dsize = wd_n->bsize;
+ ret = crypto_bin_to_hpre_bin(wd_d->data, (const char *)wd_d->data, + wd_d->bsize, wd_d->dsize, "rsa d"); + if (ret) + return ret; + wd_d->dsize = wd_d->bsize; + *data = wd_d->data;
return WD_SUCCESS; @@ -292,20 +291,19 @@ static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data) int ret;
wd_rsa_get_pubkey_params(pubkey, &wd_e, &wd_n); - ret = crypto_bin_to_hpre_bin(wd_e->data, (const char *)wd_e->data, - wd_e->bsize, wd_e->dsize, "rsa e"); - if (ret) - return ret; - wd_e->dsize = wd_e->bsize; - ret = crypto_bin_to_hpre_bin(wd_n->data, (const char *)wd_n->data, wd_n->bsize, wd_n->dsize, "rsa n"); if (ret) return ret; wd_n->dsize = wd_n->bsize;
- *data = wd_e->data; + ret = crypto_bin_to_hpre_bin(wd_e->data, (const char *)wd_e->data, + wd_e->bsize, wd_e->dsize, "rsa e"); + if (ret) + return ret; + wd_e->dsize = wd_e->bsize;
+ *data = wd_e->data; return WD_SUCCESS; }
@@ -315,19 +313,18 @@ static int fill_rsa_genkey_in(struct wd_rsa_kg_in *genkey) int ret;
wd_rsa_get_kg_in_params(genkey, &e, &q, &p); + ret = crypto_bin_to_hpre_bin(p.data, (const char *)p.data, + p.bsize, p.dsize, "rsa kg p"); + if (ret) + return ret;
ret = crypto_bin_to_hpre_bin(e.data, (const char *)e.data, e.bsize, e.dsize, "rsa kg e"); if (ret) return ret;
- ret = crypto_bin_to_hpre_bin(q.data, (const char *)q.data, + return crypto_bin_to_hpre_bin(q.data, (const char *)q.data, q.bsize, q.dsize, "rsa kg q"); - if (ret) - return ret; - - return crypto_bin_to_hpre_bin(p.data, (const char *)p.data, - p.bsize, p.dsize, "rsa kg p"); }
static int hpre_tri_bin_transfer(struct wd_dtb *bin0, struct wd_dtb *bin1, @@ -365,7 +362,7 @@ static int hpre_tri_bin_transfer(struct wd_dtb *bin0, struct wd_dtb *bin1, }
static int rsa_out_transfer(struct wd_rsa_msg *msg, - struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) + struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) { struct wd_rsa_req *req = &msg->req; struct wd_rsa_kg_out *key = req->dst; @@ -375,8 +372,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, void *data; int ret;
- if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) { - /* async */ + if (hw_msg->alg == HW_ALG_KG_CRT || hw_msg->alg == HW_ALG_KG_STD) { if (qp_mode == CTX_MODE_ASYNC) { data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); key = container_of(data, struct wd_rsa_kg_out, data); @@ -386,7 +382,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, }
msg->result = WD_SUCCESS; - if (hw_msg->alg == HPRE_ALG_KG_CRT) { + if (hw_msg->alg == HW_ALG_KG_CRT) { req->dst_bytes = CRT_GEN_PARAMS_SZ(kbytes); wd_rsa_get_kg_out_crt_params(key, &qinv, &dq, &dp); ret = hpre_tri_bin_transfer(&qinv, &dq, &dp); @@ -397,7 +393,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,
wd_rsa_set_kg_out_crt_psz(key, qinv.dsize, dq.dsize, dp.dsize); - } else if (hw_msg->alg == HPRE_ALG_KG_STD) { + } else if (hw_msg->alg == HW_ALG_KG_STD) { req->dst_bytes = GEN_PARAMS_SZ(kbytes);
wd_rsa_get_kg_out_params(key, &d, &n); @@ -423,7 +419,7 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, int ret;
if (req->op_type == WD_RSA_SIGN) { - if (hw_msg->alg == HPRE_ALG_NC_CRT) { + if (hw_msg->alg == HW_ALG_NC_CRT) { ret = fill_rsa_crt_prikey2((void *)msg->key, &data); if (ret) return ret; @@ -431,13 +427,13 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, ret = fill_rsa_prikey1((void *)msg->key, &data); if (ret) return ret; - hw_msg->alg = HPRE_ALG_NC_NCRT; + hw_msg->alg = HW_ALG_NC_NCRT; } } else if (req->op_type == WD_RSA_VERIFY) { ret = fill_rsa_pubkey((void *)msg->key, &data); if (ret) return ret; - hw_msg->alg = HPRE_ALG_NC_NCRT; + hw_msg->alg = HW_ALG_NC_NCRT; } else if (req->op_type == WD_RSA_GENKEY) { ret = fill_rsa_genkey_in((void *)msg->key); if (ret) @@ -447,10 +443,10 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, WD_ERR("failed to get rsa gen key data!\n"); return ret; } - if (hw_msg->alg == HPRE_ALG_NC_CRT) - hw_msg->alg = HPRE_ALG_KG_CRT; + if (hw_msg->alg == HW_ALG_NC_CRT) + hw_msg->alg = HW_ALG_KG_CRT; else - hw_msg->alg = HPRE_ALG_KG_STD; + hw_msg->alg = HW_ALG_KG_STD; } else { WD_ERR("invalid: rsa operatin type %u is error!\n", req->op_type); return -WD_EINVAL; @@ -525,11 +521,11 @@ out: return -WD_EINVAL; }
-static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) +static int hpre_rsa_dh_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret;
if (!config->ctx_num) { @@ -537,27 +533,19 @@ static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }
- priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V2_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv;
return WD_SUCCESS; }
-static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) +static int hpre_ecc_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret;
if (!config->ctx_num) { @@ -565,45 +553,28 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }
- priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V3_ECC_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv;
return WD_SUCCESS; }
-static void hpre_exit(struct wd_alg_driver *drv) +static void hpre_exit(void *priv) { - struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; - struct wd_ctx_config_internal *config; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; + struct wd_ctx_config_internal *config = &hpre_ctx->config; handle_t h_qp; __u32 i;
- if (!priv) { - /* return if already exit */ - return; - } - - config = &priv->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - - free(priv); - drv->priv = NULL; }
-static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_send(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_rsa_msg *msg = rsa_msg; @@ -615,9 +586,9 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
if (msg->key_type == WD_RSA_PRIKEY1 || msg->key_type == WD_RSA_PUBKEY) - hw_msg.alg = HPRE_ALG_NC_NCRT; + hw_msg.alg = HW_ALG_NC_NCRT; else if (msg->key_type == WD_RSA_PRIKEY2) - hw_msg.alg = HPRE_ALG_NC_CRT; + hw_msg.alg = HW_ALG_NC_CRT; else return -WD_EINVAL;
@@ -659,14 +630,15 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, } }
-static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_recv(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); - struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct hisi_hpre_sqe hw_msg = {0}; struct wd_rsa_msg *msg = rsa_msg; struct wd_rsa_msg *temp_msg; + struct hisi_qp *qp; __u16 recv_cnt = 0; + __u64 tag; int ret;
ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt); @@ -677,12 +649,14 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) if (ret) return ret;
- msg->tag = LW_U16(hw_msg.low_tag); + qp = (struct hisi_qp *)h_qp; + tag = LW_U16(hw_msg.low_tag); + msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_rsa_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -741,7 +715,6 @@ static int dh_out_transfer(struct wd_dh_msg *msg, void *out; int ret;
- /* async */ if (qp_mode == CTX_MODE_ASYNC) out = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); else @@ -757,7 +730,7 @@ static int dh_out_transfer(struct wd_dh_msg *msg, return WD_SUCCESS; }
-static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_send(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_dh_msg *msg = dh_msg; @@ -769,9 +742,9 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));
if (msg->is_g2 && req->op_type != WD_DH_PHASE2) - hw_msg.alg = HPRE_ALG_DH_G2; + hw_msg.alg = HW_ALG_DH_G2; else - hw_msg.alg = HPRE_ALG_DH; + hw_msg.alg = HW_ALG_DH;
hw_msg.task_len1 = msg->key_bytes / BYTE_BITS - 0x1;
@@ -802,14 +775,15 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); }
-static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_recv(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); - struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_dh_msg *msg = dh_msg; struct hisi_hpre_sqe hw_msg = {0}; struct wd_dh_msg *temp_msg; + struct hisi_qp *qp; __u16 recv_cnt = 0; + __u64 tag; int ret;
ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt); @@ -820,12 +794,14 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) if (ret) return ret;
- msg->tag = LW_U16(hw_msg.low_tag); + qp = (struct hisi_qp *)h_qp; + tag = LW_U16(hw_msg.low_tag); + msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_dh_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -852,34 +828,34 @@ static int ecc_prepare_alg(struct wd_ecc_msg *msg, { switch (msg->req.op_type) { case WD_SM2_SIGN: - hw_msg->alg = HPRE_ALG_SM2_SIGN; + hw_msg->alg = HW_ALG_SM2_SIGN; break; case WD_ECDSA_SIGN: - hw_msg->alg = HPRE_ALG_ECDSA_SIGN; + hw_msg->alg = HW_ALG_ECDSA_SIGN; break; case WD_SM2_VERIFY: - hw_msg->alg = HPRE_ALG_SM2_VERF; + hw_msg->alg = HW_ALG_SM2_VERF; break; case WD_ECDSA_VERIFY: - hw_msg->alg = HPRE_ALG_ECDSA_VERF; + hw_msg->alg = HW_ALG_ECDSA_VERF; break; case WD_SM2_ENCRYPT: - hw_msg->alg = HPRE_ALG_SM2_ENC; + hw_msg->alg = HW_ALG_SM2_ENC; break; case WD_SM2_DECRYPT: - hw_msg->alg = HPRE_ALG_SM2_DEC; + hw_msg->alg = HW_ALG_SM2_DEC; break; case WD_SM2_KG: - hw_msg->alg = HPRE_ALG_SM2_KEY_GEN; + hw_msg->alg = HW_ALG_SM2_KEY_GEN; break; case WD_ECXDH_GEN_KEY: /* fall through */ case HPRE_SM2_ENC: /* fall through */ case HPRE_SM2_DEC: /* fall through */ case WD_ECXDH_COMPUTE_KEY: if (msg->curve_id == WD_X448 || msg->curve_id == WD_X25519) - hw_msg->alg = HPRE_ALG_X_DH_MULTIPLY; + hw_msg->alg = HW_ALG_X_DH_MULTIPLY; else - hw_msg->alg = HPRE_ALG_ECDH_MULTIPLY; + hw_msg->alg = HW_ALG_ECDH_MULTIPLY; break; default: return -WD_EINVAL; @@ -957,8 +933,8 @@ static bool less_than_latter(struct wd_dtb *d, struct wd_dtb *n) ret = memcmp(d->data + shift, n->data + shift, n->dsize); if (ret < 0) return true; - else - return false; + + return false; }
static int ecc_prepare_prikey(struct wd_ecc_key *key, void **data, int id) @@ -1100,16 +1076,16 @@ static int ecc_prepare_key(struct wd_ecc_msg *msg, static void ecc_get_io_len(__u32 atype, __u32 hsz, size_t *ilen, size_t *olen) { - if (atype == HPRE_ALG_ECDH_MULTIPLY) { + if (atype == HW_ALG_ECDH_MULTIPLY) { *olen = ECDH_OUT_PARAMS_SZ(hsz); *ilen = *olen; - } else if (atype == HPRE_ALG_X_DH_MULTIPLY) { + } else if (atype == HW_ALG_X_DH_MULTIPLY) { *olen = X_DH_OUT_PARAMS_SZ(hsz); *ilen = *olen; - } else if (atype == HPRE_ALG_ECDSA_SIGN) { + } else if (atype == HW_ALG_ECDSA_SIGN) { *olen = ECC_SIGN_OUT_PARAMS_SZ(hsz); *ilen = ECC_SIGN_IN_PARAMS_SZ(hsz); - } else if (atype == HPRE_ALG_ECDSA_VERF) { + } else if (atype == HW_ALG_ECDSA_VERF) { *olen = ECC_VERF_OUT_PARAMS_SZ; *ilen = ECC_VERF_IN_PARAMS_SZ(hsz); } else { @@ -1561,31 +1537,31 @@ static int set_prikey(struct wd_ecc_prikey *prikey, struct wd_sm2_enc_in *ein = msg->req.src; int ret;
- ret = set_param(&prikey->p, &pubkey->p, "p"); + ret = set_param(&prikey->g.x, &pubkey->g.x, "gx"); if (unlikely(ret)) return ret;
- ret = set_param(&prikey->a, &pubkey->a, "a"); + ret = set_param(&prikey->b, &pubkey->b, "b"); if (unlikely(ret)) return ret;
- ret = set_param(&prikey->d, &ein->k, "k"); + ret = set_param(&prikey->a, &pubkey->a, "a"); if (unlikely(ret)) return ret;
- ret = set_param(&prikey->b, &pubkey->b, "b"); + ret = set_param(&prikey->d, &ein->k, "k"); if (unlikely(ret)) return ret;
- ret = set_param(&prikey->n, &pubkey->n, "n"); - if (unlikely(ret)) + ret = set_param(&prikey->g.y, &pubkey->g.y, "gy"); + if (ret) return ret;
- ret = set_param(&prikey->g.x, &pubkey->g.x, "gx"); + ret = set_param(&prikey->p, &pubkey->p, "p"); if (unlikely(ret)) return ret;
- return set_param(&prikey->g.y, &pubkey->g.y, "gy"); + return set_param(&prikey->n, &pubkey->n, "n"); }
static struct wd_ecc_out *create_ecdh_out(struct wd_ecc_msg *msg) @@ -1759,7 +1735,6 @@ static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) hw_msg->etype = 0x0; hw_msg->low_tag = msg->tag; hw_msg->task_len1 = hw_sz / BYTE_BITS - 0x1; - return ret; }
@@ -1893,7 +1868,7 @@ free_dst: return ret; }
-static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_send(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -1920,7 +1895,7 @@ static int ecdh_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_ms
wd_ecxdh_get_out_params(out, &key);
- if (hw_msg->alg == HPRE_ALG_ECDH_MULTIPLY) + if (hw_msg->alg == HW_ALG_ECDH_MULTIPLY) y = &key->y;
ret = hpre_tri_bin_transfer(&key->x, y, NULL); @@ -2012,26 +1987,25 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, int ret = -WD_EINVAL; void *va;
- /* async */ if (qp_mode == CTX_MODE_ASYNC) { va = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); msg->req.dst = container_of(va, struct wd_ecc_out, data); }
- if (hw_msg->alg == HPRE_ALG_SM2_SIGN || - hw_msg->alg == HPRE_ALG_ECDSA_SIGN) + if (hw_msg->alg == HW_ALG_SM2_SIGN || + hw_msg->alg == HW_ALG_ECDSA_SIGN) ret = ecc_sign_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_VERF || - hw_msg->alg == HPRE_ALG_ECDSA_VERF) + else if (hw_msg->alg == HW_ALG_SM2_VERF || + hw_msg->alg == HW_ALG_ECDSA_VERF) ret = ecc_verf_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_ENC) + else if (hw_msg->alg == HW_ALG_SM2_ENC) ret = sm2_enc_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_DEC) + else if (hw_msg->alg == HW_ALG_SM2_DEC) ret = 0; - else if (hw_msg->alg == HPRE_ALG_SM2_KEY_GEN) + else if (hw_msg->alg == HW_ALG_SM2_KEY_GEN) ret = sm2_kg_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_ECDH_MULTIPLY || - hw_msg->alg == HPRE_ALG_X_DH_MULTIPLY) + else if (hw_msg->alg == HW_ALG_ECDH_MULTIPLY || + hw_msg->alg == HW_ALG_X_DH_MULTIPLY) ret = ecdh_out_transfer(msg, hw_msg); else WD_ERR("invalid: algorithm type %u is error!\n", hw_msg->alg); @@ -2039,6 +2013,15 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, return ret; }
+static void msg_pack(char *dst, __u64 *out_len, + const void *src, __u32 src_len) +{ + if (unlikely(!src || !src_len)) + return; + + memcpy(dst + *out_len, src, src_len); + *out_len += src_len; +}
static __u32 get_hash_bytes(__u8 type) { @@ -2073,14 +2056,13 @@ static __u32 get_hash_bytes(__u8 type) return val; }
-static void msg_pack(char *dst, __u64 *out_len, - const void *src, __u32 src_len) +static void sm2_xor(struct wd_dtb *val1, struct wd_dtb *val2) { - if (unlikely(!src || !src_len)) - return; + __u32 i;
- memcpy(dst + *out_len, src, src_len); - *out_len += src_len; + for (i = 0; i < val1->dsize; ++i) + val1->data[i] = (char)((__u8)val1->data[i] ^ + (__u8)val2->data[i]); }
static int sm2_kdf(struct wd_dtb *out, struct wd_ecc_point *x2y2, @@ -2145,15 +2127,6 @@ static int sm2_kdf(struct wd_dtb *out, struct wd_ecc_point *x2y2, return ret; }
-static void sm2_xor(struct wd_dtb *val1, struct wd_dtb *val2) -{ - __u32 i; - - for (i = 0; i < val1->dsize; ++i) - val1->data[i] = (char)((__u8)val1->data[i] ^ - (__u8)val2->data[i]); -} - static int is_equal(struct wd_dtb *src, struct wd_dtb *dst) { if (src->dsize == dst->dsize && @@ -2309,14 +2282,17 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { struct wd_ecc_msg *temp_msg; + __u64 tag; int ret;
- msg->tag = LW_U16(hw_msg->low_tag); + tag = LW_U16(hw_msg->low_tag); + msg->tag = tag; + if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_ecc_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -2326,19 +2302,19 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, hpre_result_check(hw_msg, &msg->result); if (msg->result) { ret = -msg->result; - goto dump_err_msg; + goto err; }
ret = ecc_out_transfer(msg, hw_msg, qp->q_info.qp_mode); if (ret) { msg->result = WD_OUT_EPARA; WD_ERR("failed to transfer out ecc BD, ret = %d!\n", ret); - goto dump_err_msg; + goto err; }
return ret;
-dump_err_msg: +err: dump_hpre_msg(temp_msg, WD_ECC);
return ret; @@ -2373,6 +2349,7 @@ static int parse_second_sqe(handle_t h_qp, hsz = (hw_msg.task_len1 + 1) * BYTE_BITS; dst = *(struct wd_ecc_msg **)((uintptr_t)data + hsz * ECDH_OUT_PARAM_NUM); + ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg); msg->result = dst->result; *second = dst; @@ -2380,8 +2357,8 @@ static int parse_second_sqe(handle_t h_qp, return ret; }
-static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_enc_parse(handle_t h_qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *second = NULL; @@ -2464,7 +2441,7 @@ fail: return ret; }
-static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_recv(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2480,10 +2457,10 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) if (ret) return ret;
- if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && + if (hw_msg.alg == HW_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_ENC) return sm2_enc_parse(h_qp, msg, &hw_msg); - else if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && + else if (hw_msg.alg == HW_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_DEC) return sm2_dec_parse(h_qp, msg, &hw_msg);
@@ -2501,6 +2478,7 @@ static int hpre_get_usage(void *param) .alg_name = (hpre_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_hpre_ctx),\ .queue_num = HPRE_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ .fallback = 0,\ @@ -2524,6 +2502,7 @@ static struct wd_alg_driver hpre_rsa_driver = { .alg_name = "rsa", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -2539,6 +2518,7 @@ static struct wd_alg_driver hpre_dh_driver = { .alg_name = "dh", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -2566,7 +2546,7 @@ static void __attribute__((constructor)) hisi_hpre_probe(void)
ret = wd_alg_driver_register(&hpre_dh_driver); if (ret && ret != -WD_ENODEV) - WD_ERR("failed to register HPRE dh driver!\n"); + WD_ERR("failed to register HPRE rsa driver!\n");
for (i = 0; i < alg_num; i++) { ret = wd_alg_driver_register(&hpre_ecc_driver[i]); diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 304764e..30d20ff 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -8,7 +8,6 @@ #include <sys/mman.h>
#include "hisi_qm_udrv.h" -#include "wd_util.h"
#define QM_DBELL_CMD_SQ 0 #define QM_DBELL_CMD_CQ 1 @@ -25,9 +24,9 @@ #define CQE_SQ_HEAD_INDEX(cqe) (__le16_to_cpu((cqe)->sq_head) & 0xffff) #define VERSION_ID_SHIFT 9
-#define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) +#define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) #define UACCE_CMD_QM_SET_QP_INFO _IOWR('H', 11, struct hisi_qp_info) -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
/* the max sge num in one sgl */ #define HISI_SGE_NUM_IN_SGL 255 @@ -354,12 +353,12 @@ static int hisi_qm_setup_info(struct hisi_qp *qp, struct hisi_qm_priv *config) ret = pthread_spin_init(&q_info->sd_lock, PTHREAD_PROCESS_SHARED); if (ret) { WD_DEV_ERR(qp->h_ctx, "failed to init qinfo sd_lock!\n"); - goto err_destroy_lock; + goto err_destory_lock; }
return 0;
-err_destroy_lock: +err_destory_lock: pthread_spin_destroy(&q_info->rv_lock); err_out: hisi_qm_unset_region(qp->h_ctx, q_info); @@ -395,22 +394,22 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx) int ret;
if (!config) - goto out; + return (handle_t)NULL;
if (!config->sqe_size) { - WD_ERR("invalid: sqe size is zero!\n"); - goto out; + WD_DEV_ERR(ctx, "invalid: sqe size is zero!\n"); + return (handle_t)NULL; }
qp = calloc(1, sizeof(struct hisi_qp)); if (!qp) - goto out; + return (handle_t)NULL;
qp->h_ctx = ctx;
ret = hisi_qm_setup_info(qp, config); if (ret) - goto out_qp; + goto free_qp;
qp->h_sgl_pool = hisi_qm_create_sglpool(HISI_SGL_NUM_IN_BD, HISI_SGE_NUM_IN_SGL); @@ -433,9 +432,8 @@ free_pool: hisi_qm_destroy_sglpool(qp->h_sgl_pool); free_info: hisi_qm_clear_info(qp); -out_qp: +free_qp: free(qp); -out: return (handle_t)NULL; }
@@ -562,46 +560,14 @@ static int hisi_qm_recv_single(struct hisi_qm_queue_info *q_info, handle_t h_ctx return 0; }
-int hisi_qm_recv(handle_t h_qp, void *resp, __u16 expect, __u16 *count) -{ - struct hisi_qp *qp = (struct hisi_qp *)h_qp; - struct hisi_qm_queue_info *q_info; - __u16 recv_num = 0; - __u16 i; - int ret; - - if (unlikely(!resp || !qp || !count)) - return -WD_EINVAL; - - if (unlikely(!expect)) - return 0; - - q_info = &qp->q_info; - if (unlikely(wd_ioread32(q_info->ds_rx_base) == 1)) { - WD_DEV_ERR(qp->h_ctx, "wd queue hw error happened before qm receive!\n"); - return -WD_HW_EACCESS; - } - - for (i = 0; i < expect; i++) { - ret = hisi_qm_recv_single(q_info, qp->h_ctx, resp, i); - if (ret) - break; - recv_num++; - } - - *count = recv_num; - - return ret; -} - int hisi_check_bd_id(handle_t h_qp, __u32 mid, __u32 bid) { struct hisi_qp *qp = (struct hisi_qp *)h_qp; __u8 mode = qp->q_info.qp_mode;
if (mode == CTX_MODE_SYNC && mid != bid) { - WD_DEV_ERR(qp->h_ctx, "failed to recv self bd, send id: %u, recv id: %u\n", - mid, bid); + WD_DEV_ERR(qp->h_ctx, "failed to recv bd, send id:%u, recv id:%u!\n", + mid, bid); return -WD_EINVAL; }
@@ -630,6 +596,39 @@ void hisi_set_msg_id(handle_t h_qp, __u32 *tag) } }
+int hisi_qm_recv(handle_t h_qp, void *resp, __u16 expect, __u16 *count) +{ + struct hisi_qp *qp = (struct hisi_qp *)h_qp; + struct hisi_qm_queue_info *q_info; + __u16 recv_num = 0; + __u16 i; + int ret; + + if (unlikely(!resp || !qp || !count)) + return -WD_EINVAL; + + if (unlikely(!expect)) + return 0; + + q_info = &qp->q_info; + + if (unlikely(wd_ioread32(q_info->ds_rx_base) == 1)) { + WD_DEV_ERR(qp->h_ctx, "wd queue hw error happened before qm receive!\n"); + return -WD_HW_EACCESS; + } + + for (i = 0; i < expect; i++) { + ret = hisi_qm_recv_single(q_info, qp->h_ctx, resp, i); + if (ret) + break; + recv_num++; + } + + *count = recv_num; + + return ret; +} + static void *hisi_qm_create_sgl(__u32 sge_num) { void *sgl; @@ -745,7 +744,9 @@ void hisi_qm_destroy_sglpool(handle_t sgl_pool) }
pool = (struct hisi_sgl_pool *)sgl_pool; + pthread_spin_destroy(&pool->lock); + hisi_qm_free_sglpool(pool); }
diff --git a/drv/hisi_qm_udrv.h b/drv/hisi_qm_udrv.h index b02e8e7..0220e89 100644 --- a/drv/hisi_qm_udrv.h +++ b/drv/hisi_qm_udrv.h @@ -6,11 +6,13 @@
#include <stdbool.h> #include <stddef.h> +#include <stdint.h> #include <pthread.h> #include <linux/types.h>
-#include "config.h" -#include "wd_util.h" +#include "uacce.h" +#include "wd.h" +#include "wd_alg_common.h"
#ifdef __cplusplus extern "C" { @@ -86,7 +88,6 @@ struct hisi_qp { struct hisi_qm_queue_info q_info; handle_t h_sgl_pool; handle_t h_ctx; - /* Private area for driver use, point to queue specifial data */ void *priv; };
@@ -126,7 +127,7 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx); void hisi_qm_free_qp(handle_t h_qp);
/** - * hisi_check_bd_id - Check the SQE BD's id and send msg id. + * hisi_check_bd_id - Check the SQE BD's id and send msg id * @h_qp: Handle of the qp. * @mid: send message id. * @bid: recv BD id. @@ -134,7 +135,7 @@ void hisi_qm_free_qp(handle_t h_qp); int hisi_check_bd_id(handle_t h_qp, __u32 mid, __u32 bid);
/** - * hisi_set_msg_id - set the message tag id. + * hisi_set_msg_id - set the alg message tag id * @h_qp: Handle of the qp. * @tag: the message tag id. */ diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 0a1bcc1..f008c16 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1,15 +1,15 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */
-#include <stdbool.h> -#include <stdlib.h> -#include <pthread.h> #include "drv/wd_cipher_drv.h" #include "drv/wd_digest_drv.h" #include "drv/wd_aead_drv.h" #include "crypto/aes.h" #include "crypto/galois.h" #include "hisi_qm_udrv.h" +#include "wd_cipher.h" +#include "wd_digest.h" +#include "wd_aead.h"
#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 @@ -523,76 +523,76 @@ static __u32 g_sec_hmac_full_len[WD_DIGEST_TYPE_MAX] = { SEC_HMAC_SHA512_MAC_LEN, SEC_HMAC_SHA512_224_MAC_LEN, SEC_HMAC_SHA512_256_MAC_LEN };
-static int hisi_sec_init(struct wd_alg_driver *drv, void *conf); -static void hisi_sec_exit(struct wd_alg_driver *drv); +static int hisi_sec_init(void *conf, void *priv); +static void hisi_sec_exit(void *priv);
-static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg);
-static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg);
-static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg);
-static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_send(drv, ctx, msg); - return hisi_sec_cipher_send_v3(drv, ctx, msg); + return hisi_sec_cipher_send(ctx, msg); + return hisi_sec_cipher_send_v3(ctx, msg); }
-static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_recv(drv, ctx, msg); - return hisi_sec_cipher_recv_v3(drv, ctx, msg); + return hisi_sec_cipher_recv(ctx, msg); + return hisi_sec_cipher_recv_v3(ctx, msg); }
-static int digest_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_send(drv, ctx, msg); - return hisi_sec_digest_send_v3(drv, ctx, msg); + return hisi_sec_digest_send(ctx, msg); + return hisi_sec_digest_send_v3(ctx, msg); }
-static int digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_recv(drv, ctx, msg); - return hisi_sec_digest_recv_v3(drv, ctx, msg); + return hisi_sec_digest_recv(ctx, msg); + return hisi_sec_digest_recv_v3(ctx, msg); }
-static int aead_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_send(drv, ctx, msg); - return hisi_sec_aead_send_v3(drv, ctx, msg); + return hisi_sec_aead_send(ctx, msg); + return hisi_sec_aead_send_v3(ctx, msg); }
-static int aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);
if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_recv(drv, ctx, msg); - return hisi_sec_aead_recv_v3(drv, ctx, msg); + return hisi_sec_aead_recv(ctx, msg); + return hisi_sec_aead_recv_v3(ctx, msg); }
static int hisi_sec_get_usage(void *param) @@ -606,6 +606,7 @@ static int hisi_sec_get_usage(void *param) .alg_name = (sec_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_sec_ctx),\ .queue_num = SEC_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ .fallback = 0,\ @@ -1141,7 +1142,7 @@ static int fill_cipher_bd2(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) return 0; }
-static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1167,6 +1168,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w }
hisi_set_msg_id(h_qp, &msg->tag); + sqe.type2.clen_ivhlen |= (__u32)msg->in_bytes; sqe.type2.tag = (__u16)msg->tag; fill_cipher_bd2_addr(msg, &sqe); @@ -1186,7 +1188,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w return 0; }
-static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1339,12 +1341,13 @@ static int fill_cipher_bd3(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) return ret; }
- sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; + sqe->auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET);
return 0; }
-static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1370,6 +1373,7 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void }
hisi_set_msg_id(h_qp, &msg->tag); + sqe.c_len_ivin = (__u32)msg->in_bytes; sqe.tag = (__u64)(uintptr_t)msg->tag; fill_cipher_bd3_addr(msg, &sqe); @@ -1377,7 +1381,7 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("cipher send sqe is err(%d)!\n", ret); + WD_ERR("cipher send sqe v3 is err(%d)!\n", ret);
if (msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, @@ -1434,7 +1438,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "cipher"); }
-static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1688,7 +1692,7 @@ static int digest_len_check(struct wd_digest_msg *msg, enum sec_bd_type type) return 0; }
-static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -1755,7 +1759,7 @@ put_sgl: return ret; }
-static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -1932,7 +1936,7 @@ static void fill_digest_v3_scene(struct hisi_sec_sqe3 *sqe, sqe->bd_param |= (__u16)(de | scene); }
-static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -1975,12 +1979,13 @@ static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void
hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = (__u64)(uintptr_t)msg->tag; - sqe.auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; + sqe.auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET);
ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("digest send sqe is err(%d)!\n", ret); + WD_ERR("digest send sqe v3 is err(%d)!\n", ret);
goto put_sgl; } @@ -2031,7 +2036,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "digest"); }
-static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -2524,7 +2529,7 @@ int aead_msg_state_check(struct wd_aead_msg *msg) return 0; }
-static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *msg = wd_msg; @@ -2647,13 +2652,22 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) { /* Asynchronous mode does not use the sent message, so ignores it */ - if (msg->cmode == WD_CIPHER_GCM) - return (msg->msg_state == AEAD_MSG_END) && qp->q_info.qp_mode == CTX_MODE_SYNC; + if (qp->q_info.qp_mode == CTX_MODE_ASYNC) + return false; + /* + * For aead gcm stream mode, due to some hardware limitations, + * the final message was not sent to hardware if the qm is + * not higher than v3 version or the input length of the + * message is 0, the software calculation has been executed. + */ + if (msg->msg_state == AEAD_MSG_END && msg->cmode == WD_CIPHER_GCM && + (qp->q_info.hw_type <= HISI_QM_API_VER3_BASE || !msg->in_bytes)) + return true;
return false; }
-static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -2903,7 +2917,6 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) sqe->c_len_ivin = msg->in_bytes; sqe->cipher_src_offset = msg->assoc_bytes; sqe->a_len_key = msg->in_bytes + msg->assoc_bytes; - sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET;
ret = fill_aead_bd3_alg(msg, sqe); if (ret) { @@ -2917,10 +2930,13 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) return ret; }
+ sqe->auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET); + return 0; }
-static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *msg = wd_msg; @@ -2954,6 +2970,7 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * }
fill_aead_bd3_addr(msg, &sqe); + ret = fill_stream_bd3(h_qp, msg, &sqe); if (ret == WD_SOFT_COMPUTING) { ret = 0; @@ -2967,7 +2984,7 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("aead send sqe is err(%d)!\n", ret); + WD_ERR("aead send sqe v3 is err(%d)!\n", ret);
goto put_sgl; } @@ -3024,7 +3041,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "aead"); }
-static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3052,11 +3069,11 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * return 0; }
-static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) +static int hisi_sec_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_sec_ctx *sec_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_sec_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -3066,10 +3083,6 @@ static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }
- priv = malloc(sizeof(struct hisi_sec_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { @@ -3086,8 +3099,7 @@ static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) goto out; config->ctxs[i].sqn = qm_priv.sqn; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sec_ctx->config, config, sizeof(struct wd_ctx_config_internal));
return 0;
@@ -3096,29 +3108,26 @@ out: h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); hisi_qm_free_qp(h_qp); } - free(priv); return -WD_EINVAL; }
-static void hisi_sec_exit(struct wd_alg_driver *drv) +static void hisi_sec_exit(void *priv) { - struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; + struct hisi_sec_ctx *sec_ctx = priv; struct wd_ctx_config_internal *config; handle_t h_qp; __u32 i;
if (!priv) { - /* return if already exit */ + WD_ERR("invalid: input parameter is NULL!\n"); return; }
- config = &priv->config; + config = &sec_ctx->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; }
#ifdef WD_STATIC_DRV @@ -3179,3 +3188,4 @@ static void __attribute__((destructor)) hisi_sec2_remove(void) for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&aead_alg_driver[i]); } + diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 99cd640..c8812df 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -22,10 +22,10 @@ typedef void (sm3_ce_block_fn)(__u32 word_reg[SM3_STATE_WORDS], const unsigned char *src, size_t blocks);
-static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf); -static void sm3_ce_drv_exit(struct wd_alg_driver *drv); -static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); -static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); +static int sm3_ce_drv_init(void *conf, void *priv); +static void sm3_ce_drv_exit(void *priv); +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg); +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg); static int sm3_ce_get_usage(void *param);
static struct wd_alg_driver sm3_ce_alg_driver = { @@ -33,6 +33,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, .priority = 200, + .priv_size = sizeof(struct sm3_ce_drv_ctx), .queue_num = 1, .op_type_num = 1, .fallback = 0, @@ -337,7 +338,7 @@ static int do_hmac_sm3_ce(struct wd_digest_msg *msg, __u8 *out_hmac) return WD_SUCCESS; }
-static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) { struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; __u8 digest[SM3_DIGEST_SIZE] = {0}; @@ -365,38 +366,26 @@ static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest return ret; }
-static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg) { return WD_SUCCESS; }
-static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf) +static int sm3_ce_drv_init(void *conf, void *priv) { - struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; - struct sm3_ce_drv_ctx *priv; + struct wd_ctx_config_internal *config = conf; + struct sm3_ce_drv_ctx *sctx = priv;
/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;
- priv = malloc(sizeof(struct sm3_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal));
return WD_SUCCESS; }
-static void sm3_ce_drv_exit(struct wd_alg_driver *drv) +static void sm3_ce_drv_exit(void *priv) { - struct sm3_ce_drv_ctx *sctx = (struct sm3_ce_drv_ctx *)drv->priv; - - if (!sctx) - return; - - free(sctx); - drv->priv = NULL; } diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 3404465..6475246 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -8,12 +8,12 @@ * https://www.openssl.org/source/license.html */ /* - * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */
#include "drv/wd_cipher_drv.h" -#include "wd_cipher.h" #include "isa_ce_sm4.h" +#include "wd_cipher.h"
#define SM4_ENCRYPT 1 #define SM4_DECRYPT 0 @@ -31,35 +31,23 @@ ((p)[0] = (__u8)((v) >> 24), (p)[1] = (__u8)((v) >> 16), \ (p)[2] = (__u8)((v) >> 8), (p)[3] = (__u8)(v))
-static int isa_ce_init(struct wd_alg_driver *drv, void *conf) +static int isa_ce_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct sm4_ce_drv_ctx *priv; + struct sm4_ce_drv_ctx *sctx = priv;
/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;
- priv = malloc(sizeof(struct sm4_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal));
- return WD_SUCCESS; + return 0; }
-static void isa_ce_exit(struct wd_alg_driver *drv) +static void isa_ce_exit(void *priv) { - struct sm4_ce_drv_ctx *sctx = (struct sm4_ce_drv_ctx *)drv->priv; - - if (!sctx) - return; - - free(sctx); - drv->priv = NULL; }
/* increment upper 96 bits of 128-bit counter by 1 */ @@ -333,7 +321,7 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey return 0; }
-static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; @@ -399,19 +387,19 @@ static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ return ret; }
-static int isa_ce_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { return 0; }
-static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { - return isa_ce_cipher_send(drv, ctx, msg); + return isa_ce_cipher_send(ctx, msg); }
-static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { - return isa_ce_cipher_recv(drv, ctx, msg); + return isa_ce_cipher_recv(ctx, msg); }
#define GEN_CE_ALG_DRIVER(ce_alg_name, alg_type) \ @@ -420,6 +408,7 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ .priority = 200,\ + .priv_size = sizeof(struct sm4_ce_drv_ctx),\ .op_type_num = 1,\ .fallback = 0,\ .init = isa_ce_init,\ diff --git a/include/wd.h b/include/wd.h index 21e5e06..77aac72 100644 --- a/include/wd.h +++ b/include/wd.h @@ -35,13 +35,6 @@ extern "C" { typedef unsigned char __u8; typedef unsigned int __u32; typedef unsigned long long __u64; -/* Required compiler attributes */ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -#define handle_t uintptr_t -typedef struct wd_dev_mask wd_dev_mask_t; - typedef void (*wd_log)(const char *format, ...);
#ifndef WD_NO_LOG @@ -104,12 +97,7 @@ typedef void (*wd_log)(const char *format, ...); #define WD_IS_ERR(h) ((uintptr_t)(h) > \ (uintptr_t)(-1000))
-enum wd_buff_type { - WD_FLAT_BUF, - WD_SGL_BUF, -}; - -enum wd_alg_type { +enum wcrypto_type { WD_CIPHER, WD_DIGEST, WD_AEAD, @@ -155,6 +143,9 @@ struct wd_dev_mask { unsigned int magic; };
+#define handle_t uintptr_t +typedef struct wd_dev_mask wd_dev_mask_t; + #if defined(__AARCH64_CMODEL_SMALL__) && __AARCH64_CMODEL_SMALL__ #define dsb(opt) { asm volatile("dsb " #opt : : : "memory"); } #define rmb() dsb(ld) /* read fence */ @@ -194,14 +185,14 @@ static inline void wd_iowrite64(void *addr, uint64_t value) *((volatile uint64_t *)addr) = value; }
-static inline void *WD_ERR_PTR(uintptr_t error) +static inline void *WD_ERR_PTR(intptr_t error) { return (void *)error; }
static inline long WD_PTR_ERR(const void *ptr) { - return (long)ptr; + return (intptr_t)ptr; }
/** diff --git a/include/wd_aead.h b/include/wd_aead.h index 01f6980..c17cd1d 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -8,6 +8,7 @@ #define __WD_AEAD_H
#include <dlfcn.h> +#include "wd.h" #include "wd_alg_common.h" #include "wd_cipher.h" #include "wd_digest.h" diff --git a/include/wd_alg.h b/include/wd_alg.h index aba855d..6203893 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -62,7 +62,7 @@ extern "C" { # define HWCAP2_RNG (1 << 16) #endif
-enum alg_dev_type { +enum alg_priority { UADK_ALG_SOFT = 0x0, UADK_ALG_CE_INSTR = 0x1, UADK_ALG_SVE_INSTR = 0x2, @@ -104,38 +104,18 @@ struct wd_alg_driver { int calc_type; int queue_num; int op_type_num; - void *priv; + int priv_size; handle_t fallback;
- int (*init)(struct wd_alg_driver *drv, void *conf); - void (*exit)(struct wd_alg_driver *drv); - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*init)(void *conf, void *priv); + void (*exit)(void *priv); + int (*send)(handle_t ctx, void *drv_msg); + int (*recv)(handle_t ctx, void *drv_msg); int (*get_usage)(void *param); int (*get_extend_ops)(void *ops); };
-inline int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) -{ - return drv->init(drv, conf); -} - -inline void wd_alg_driver_exit(struct wd_alg_driver *drv) -{ - drv->exit(drv); -} - -inline int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->send(drv, ctx, msg); -} - -inline int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->recv(drv, ctx, msg); -} - -/* +/** * wd_alg_driver_register() - Register a device driver. * @wd_alg_driver: a device driver that supports an algorithm. * diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index fd77426..d1c0038 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -10,6 +10,8 @@ #include <pthread.h> #include <stdbool.h> #include <numa.h> +#include <asm/types.h> + #include "wd.h" #include "wd_alg.h"
@@ -17,10 +19,11 @@ extern "C" { #endif
-#define BYTE_BITS 8 -#define BYTE_BITS_SHIFT 3 -#define GET_NEGATIVE(val) (0 - (val)) +/* Required compiler attributes */ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0)
+#define BYTE_BITS_SHIFT 3 #define BITS_TO_BYTES(bits) (((bits) + 7) >> 3) #define BYTES_TO_BITS(bytes) ((bytes) << 3)
@@ -55,12 +58,17 @@ enum wd_ctx_mode { CTX_MODE_MAX, };
+enum wd_buff_type { + WD_FLAT_BUF, + WD_SGL_BUF, +}; + enum wd_init_type { WD_TYPE_V1, WD_TYPE_V2, };
-/* +/** * struct wd_ctx - Define one ctx and related type. * @ctx: The ctx itself. * @op_type: Define the operation type of this specific ctx. @@ -74,7 +82,7 @@ struct wd_ctx { __u8 ctx_mode; };
-/* +/** * struct wd_cap_config - Capabilities. * @ctx_msg_num: number of asynchronous msg pools that the user wants to allocate. * Optional, user can set ctx_msg_num based on the number of requests @@ -87,7 +95,7 @@ struct wd_cap_config { __u32 resv; };
-/* +/** * struct wd_ctx_config - Define a ctx set and its related attributes, which * will be used in the scope of current process. * @ctx_num: The ctx number in below ctx array. @@ -103,7 +111,7 @@ struct wd_ctx_config { struct wd_cap_config *cap; };
-/* +/** * struct wd_ctx_nums - Define the ctx sets numbers. * @sync_ctx_num: The ctx numbers which are used for sync mode for each * ctx sets. @@ -115,7 +123,7 @@ struct wd_ctx_nums { __u32 async_ctx_num; };
-/* +/** * struct wd_ctx_params - Define the ctx sets params which are used for init * algorithms. * @op_type_num: Used for index of ctx_set_num, the order is the same as @@ -153,11 +161,11 @@ struct wd_ctx_config_internal { unsigned long *msg_cnt; };
-/* +/** * struct wd_comp_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling - * @sched_init: inited the scheduler input parameters. + * @sched_init: inited the scheduler input parameters. * @pick_next_ctx: Pick the proper ctx which a request will be sent to. * config points to the ctx config; sched_ctx points to * scheduler context; req points to the request. Return diff --git a/include/wd_comp.h b/include/wd_comp.h index 45994ff..c5dcefa 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -7,8 +7,7 @@ #ifndef __WD_COMP_H #define __WD_COMP_H
-#include <numa.h> - +#include "wd.h" #include "wd_alg_common.h"
#ifdef __cplusplus diff --git a/include/wd_digest.h b/include/wd_digest.h index f0916c3..d88a66c 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -8,6 +8,7 @@ #define __WD_DIGEST_H #include <dlfcn.h>
+#include "wd.h" #include "wd_alg_common.h"
#ifdef __cplusplus @@ -144,12 +145,12 @@ struct wd_digest_req { };
struct wd_cb_tag { - void *ctx; /* user: context or other user relatives */ - void *tag; /* to store user tag */ - int ctx_id; /* user id: context ID or other user identifier */ + void *ctx; /* user: context or other user relatives */ + void *tag; /* to store user tag */ + int ctx_id; /* user id: context ID or other user identifier */ };
-/* Digest tag format */ +/* Digest tag format of warpdrive */ struct wd_digest_tag { struct wd_cb_tag wd_tag; __u64 long_data_len; diff --git a/include/wd_util.h b/include/wd_util.h index dc1e41f..63aa5af 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -16,16 +16,17 @@ #include "wd.h" #include "wd_sched.h" #include "wd_alg.h" +#include "wd_alg_common.h"
#ifdef __cplusplus extern "C" { #endif
-#define WD_POOL_MAX_ENTRIES 1024 +#define WD_POOL_MAX_ENTRIES 1024
#define FOREACH_NUMA(i, config, config_numa) \ for ((i) = 0, (config_numa) = (config)->config_per_numa; \ - (i) < (config)->numa_num; (config_numa)++, (i)++) + (i) < (config)->numa_num; (config_numa)++, (i)++)
enum wd_status { WD_UNINIT, @@ -118,8 +119,8 @@ struct wd_ctx_attr { };
struct wd_msg_handle { - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*send)(handle_t sess, void *msg); + int (*recv)(handle_t sess, void *msg); };
struct wd_init_attrs { @@ -376,7 +377,6 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en);
/** * wd_handle_msg_sync() - recv msg from hardware - * @drv: the driver to handle msg. * @msg_handle: callback of msg handle ops. * @ctx: the handle of context. * @msg: the msg of task. @@ -385,8 +385,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); * * Return 0 if successful or less than 0 otherwise. */ -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en); +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en);
/** * wd_init_check() - Check input parameters for wd_<alg>_init. @@ -443,11 +443,11 @@ static inline void wd_alg_clear_init(enum wd_status *status) /** * wd_ctx_param_init() - Initialize the current device driver according * to the obtained queue resource and the applied driver. - * @ctx_params: wd_ctx_params to be initialized. - * @user_ctx_params: user input wd_ctx_params. - * @driver: device driver for the current algorithm application. + * @ctx_params: queue configuration parameters required for device initialization. + * @user_ctx_params: user-configured queue context parameters. + * @driver: device drivers that support current business algorithms. * @type: algorithm type. - * @max_op_type: algorithm max operation type. + * @max_op_type: number of device queue types that support algorithmic services. * * Return 0 if succeed and other error number if fail. */ @@ -484,13 +484,14 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv); * to the obtained queue resource and the applied driver. * @config: device resources requested by the current algorithm. * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. * * Return 0 if succeed and other error number if fail. */ int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); + struct wd_alg_driver *driver, void **drv_priv); void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); + struct wd_alg_driver *driver, void **drv_priv);
/** * wd_dlopen_drv() - Open the dynamic library file of the device driver. diff --git a/wd.c b/wd.c index 6d01158..2da5a5f 100644 --- a/wd.c +++ b/wd.c @@ -18,8 +18,10 @@ #include <numa.h> #include <sched.h>
-#include "wd.h" +#include "wd_alg_common.h" #include "wd_alg.h" +#include "wd.h" + #define SYS_CLASS_DIR "/sys/class/uacce" #define FILE_MAX_SIZE (8 << 20)
@@ -202,7 +204,7 @@ int wd_is_isolate(struct uacce_dev *dev) int value = 0; int ret;
- if (!dev || !strlen(dev->dev_root)) + if (!dev) return -WD_EINVAL;
ret = access_attr(dev->dev_root, "isolate", F_OK); @@ -282,12 +284,12 @@ static struct uacce_dev *read_uacce_sysfs(const char *dev_name) return NULL;
ret = snprintf(dev->dev_root, MAX_DEV_NAME_LEN, "%s/%s", - SYS_CLASS_DIR, dev_name); + SYS_CLASS_DIR, dev_name); if (ret < 0) goto out;
ret = snprintf(dev->char_dev_path, MAX_DEV_NAME_LEN, - "/dev/%s", dev_name); + "/dev/%s", dev_name); if (ret < 0) goto out;
@@ -703,7 +705,7 @@ struct uacce_dev_list *wd_get_accel_list(const char *alg_name) ret = access_attr(SYS_CLASS_DIR, dev_dir->d_name, F_OK); if (ret < 0) { WD_ERR("failed to access dev: %s, ret: %d\n", - dev_dir->d_name, ret); + dev_dir->d_name, ret); continue; }
diff --git a/wd_alg.c b/wd_alg.c index 3d014e1..5c5c665 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -5,9 +5,9 @@
#define _GNU_SOURCE #include <dirent.h> -#include <errno.h> #include <stdbool.h> #include <stdlib.h> +#include <string.h> #include <pthread.h> #include <sys/auxv.h>
diff --git a/wd_cipher.c b/wd_cipher.c index f6b035a..a833074 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -53,6 +53,7 @@ struct wd_cipher_setting { struct wd_sched sched; struct wd_async_msg_pool pool; struct wd_alg_driver *driver; + void *priv; void *dlhandle; void *dlh_list; } wd_cipher_setting; @@ -346,7 +347,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, goto out_clear_sched;
ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + wd_cipher_setting.driver, + &wd_cipher_setting.priv); if (ret) goto out_clear_pool;
@@ -363,10 +365,9 @@ out_clear_ctx_config:
static int wd_cipher_common_uninit(void) { - enum wd_status status; + void *priv = wd_cipher_setting.priv;
- wd_alg_get_init(&wd_cipher_setting.status, &status); - if (status == WD_UNINIT) + if (!priv) return -WD_EINVAL;
/* uninit async request pool */ @@ -376,7 +377,8 @@ static int wd_cipher_common_uninit(void) wd_clear_sched(&wd_cipher_setting.sched);
wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + wd_cipher_setting.driver, + &wd_cipher_setting.priv);
return 0; } @@ -588,9 +590,10 @@ static int cipher_iv_len_check(struct wd_cipher_req *req, return ret; }
-static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) +static int cipher_in_len_check(handle_t h_sess, struct wd_cipher_req *req) { struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + int ret = 0;
if (!req->in_bytes) { WD_ERR("invalid: cipher input length is zero!\n"); @@ -600,14 +603,27 @@ static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) if (sess->alg != WD_CIPHER_AES && sess->alg != WD_CIPHER_SM4) return 0;
- if ((req->in_bytes & (AES_BLOCK_SIZE - 1)) && - (sess->mode == WD_CIPHER_CBC || sess->mode == WD_CIPHER_ECB)) { - WD_ERR("failed to check input bytes of AES or SM4, size = %u\n", - req->in_bytes); - return -WD_EINVAL; + switch (sess->mode) { + case WD_CIPHER_ECB: + case WD_CIPHER_CBC: + if (req->in_bytes & (AES_BLOCK_SIZE - 1)) + ret = -WD_EINVAL; + break; + case WD_CIPHER_CBC_CS1: + case WD_CIPHER_CBC_CS2: + case WD_CIPHER_CBC_CS3: + if (req->in_bytes < AES_BLOCK_SIZE) + ret = -WD_EINVAL; + break; + default: + break; }
- return 0; + if (ret) + WD_ERR("invalid: %s input bytes is %u!\n", + wd_cipher_alg_name[sess->alg][sess->mode], req->in_bytes); + + return ret; }
static int wd_cipher_check_params(handle_t h_sess, @@ -632,7 +648,7 @@ static int wd_cipher_check_params(handle_t h_sess, return -WD_EINVAL; }
- ret = cipher_len_check(h_sess, req); + ret = cipher_in_len_check(h_sess, req); if (unlikely(ret)) return ret;
@@ -672,8 +688,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, msg_handle.recv = wd_cipher_setting.driver->recv;
wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, + wd_cipher_setting.config.epoll_en); wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type);
return ret; @@ -748,7 +764,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id;
- ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = wd_cipher_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -797,7 +813,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;
do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = wd_cipher_setting.driver->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { diff --git a/wd_comp.c b/wd_comp.c index 0fa5f92..9d25ce5 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -12,7 +12,10 @@ #include <string.h> #include <time.h>
+#include "config.h" #include "drv/wd_comp_drv.h" +#include "wd_sched.h" +#include "wd_util.h" #include "wd_comp.h"
#define HW_CTX_SIZE (64 * 1024) @@ -47,6 +50,7 @@ struct wd_comp_setting { struct wd_sched sched; struct wd_async_msg_pool pool; struct wd_alg_driver *driver; + void *priv; void *dlhandle; void *dlh_list; } wd_comp_setting; @@ -146,8 +150,7 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc { int ret;
- ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", - &wd_comp_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", &wd_comp_setting.config.epoll_en); if (ret < 0) return ret;
@@ -166,7 +169,8 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc goto out_clear_sched;
ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_comp_setting.driver, + &wd_comp_setting.priv); if (ret) goto out_clear_pool;
@@ -183,10 +187,9 @@ out_clear_ctx_config:
static int wd_comp_uninit_nolock(void) { - enum wd_status status; + void *priv = wd_comp_setting.priv;
- wd_alg_get_init(&wd_comp_setting.status, &status); - if (status == WD_UNINIT) + if (!priv) return -WD_EINVAL;
/* Uninit async request pool */ @@ -196,7 +199,8 @@ static int wd_comp_uninit_nolock(void) wd_clear_sched(&wd_comp_setting.sched);
wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_comp_setting.driver, + &wd_comp_setting.priv);
return 0; } @@ -377,7 +381,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;
do { - ret = wd_alg_driver_recv(wd_comp_setting.driver, ctx->ctx, &resp_msg); + ret = wd_comp_setting.driver->recv(ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -611,8 +615,8 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, msg_handle.recv = wd_comp_setting.driver->recv;
pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_comp_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock);
return ret; @@ -866,12 +870,18 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS;
- ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); + pthread_spin_lock(&ctx->lock); + + ret = wd_comp_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { - WD_ERR("wd comp send error, ret = %d!\n", ret); + pthread_spin_unlock(&ctx->lock); + if (ret != -WD_EBUSY) + WD_ERR("wd comp send error, ret = %d!\n", ret); goto fail_with_msg; }
+ pthread_spin_unlock(&ctx->lock); + wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_comp_env_config, idx); if (unlikely(ret)) diff --git a/wd_digest.c b/wd_digest.c index 58f621a..9fb9c9b 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -7,11 +7,12 @@ #include <stdlib.h> #include <pthread.h> #include <limits.h> +#include "wd_util.h" #include "include/drv/wd_digest_drv.h" #include "wd_digest.h"
#define GMAC_IV_LEN 16 -#define MAX_BLOCK_SIZE 128 +#define MAX_BLOCK_SIZE 128U
static __u32 g_digest_mac_len[WD_DIGEST_TYPE_MAX] = { WD_DIGEST_SM3_LEN, WD_DIGEST_MD5_LEN, WD_DIGEST_SHA1_LEN, @@ -42,6 +43,7 @@ struct wd_digest_setting { struct wd_sched sched; struct wd_alg_driver *driver; struct wd_async_msg_pool pool; + void *priv; void *dlhandle; void *dlh_list; } wd_digest_setting; @@ -277,7 +279,8 @@ static int wd_digest_init_nolock(struct wd_ctx_config *config, goto out_clear_sched;
ret = wd_alg_init_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + wd_digest_setting.driver, + &wd_digest_setting.priv); if (ret) goto out_clear_pool;
@@ -326,29 +329,21 @@ out_clear_init: return ret; }
-static int wd_digest_uninit_nolock(void) +static void wd_digest_uninit_nolock(void) { - enum wd_status status; - - wd_alg_get_init(&wd_digest_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_digest_setting.pool); wd_clear_sched(&wd_digest_setting.sched); wd_alg_uninit_driver(&wd_digest_setting.config, - wd_digest_setting.driver); - return 0; + wd_digest_setting.driver, + &wd_digest_setting.priv); }
void wd_digest_uninit(void) { - int ret; - - ret = wd_digest_uninit_nolock(); - if (ret) + if (!wd_digest_setting.priv) return;
+ wd_digest_uninit_nolock(); wd_digest_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_digest_setting.status); } @@ -391,7 +386,6 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, if (state) goto out_uninit;
- while (ret != 0) { memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal));
@@ -450,12 +444,10 @@ out_uninit:
void wd_digest_uninit2(void) { - int ret; - - ret = wd_digest_uninit_nolock(); - if (ret) + if (!wd_digest_setting.priv) return;
+ wd_digest_uninit_nolock(); wd_alg_attrs_uninit(&wd_digest_init_attrs); wd_alg_drv_unbind(wd_digest_setting.driver); wd_digest_close_driver(WD_TYPE_V2); @@ -618,8 +610,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds msg_handle.recv = wd_digest_setting.driver->recv;
wd_ctx_spin_lock(ctx, wd_digest_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_digest_setting.config.epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, wd_digest_setting.config.epoll_en); wd_ctx_spin_unlock(ctx, wd_digest_setting.driver->calc_type); if (unlikely(ret)) return ret; @@ -713,7 +705,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) fill_request_msg(msg, req, dsess); msg->tag = msg_id;
- ret = wd_alg_driver_send(wd_digest_setting.driver, ctx->ctx, msg); + ret = wd_digest_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -762,7 +754,8 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;
do { - ret = wd_alg_driver_recv(wd_digest_setting.driver, ctx->ctx, &recv_msg); + ret = wd_digest_setting.driver->recv(ctx->ctx, + &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { diff --git a/wd_sched.c b/wd_sched.c index aa6c91e..36bf71e 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -499,7 +499,8 @@ static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) } }
-int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) +int wd_sched_rr_instance(const struct wd_sched *sched, + struct sched_params *param) { struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; diff --git a/wd_util.c b/wd_util.c index d58e8b8..2dd81c9 100644 --- a/wd_util.c +++ b/wd_util.c @@ -24,11 +24,10 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b))
#define WD_INIT_SLEEP_UTIME 1000 -#define WD_INIT_RETRY_TIMES 10000 +#define WD_INIT_RETRY_TIMES 1000 #define US2S(us) ((us) >> 20) #define WD_INIT_RETRY_TIMEOUT 3 - -#define WD_SOFT_CTX_NUM 2 +#define WD_SOFT_CTX_NUM 2 #define WD_SOFT_SYNC_CTX 0 #define WD_SOFT_ASYNC_CTX 1
@@ -421,8 +420,15 @@ void wd_uninit_async_request_pool(struct wd_async_msg_pool *pool) void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, int ctx_idx, __u32 tag) { - struct msg_pool *p = &pool->pools[ctx_idx]; - __u32 msg_num = p->msg_num; + struct msg_pool *p; + __u32 msg_num; + + if ((__u32)ctx_idx > pool->pool_num) { + WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); + return NULL; + } + p = &pool->pools[ctx_idx]; + msg_num = p->msg_num;
/* tag value start from 1 */ if (tag == 0 || tag > msg_num) { @@ -1585,17 +1591,22 @@ static int wd_init_async_polling_thread_per_numa(struct wd_env_config *config, task_queue = queue_head; for (i = 0; i < num; task_queue++, i++) { ret = wd_init_one_task_queue(task_queue, alg_poll_ctx); - if (ret) { - for (j = 0; j < i; task_queue++, j++) - wd_uninit_one_task_queue(task_queue); - free(queue_head); - return ret; - } + if (ret) + goto uninit; }
config_numa->async_task_queue_array = (void *)queue_head;
return 0; + +uninit: + task_queue = queue_head; + for (j = 0; j < i; task_queue++, j++) + wd_uninit_one_task_queue(task_queue); + + free(queue_head); + + return ret; }
static void wd_uninit_async_polling_thread_per_numa(struct wd_env_config *cfg, @@ -1814,8 +1825,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en) return 0; }
-int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en) +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en) { __u64 timeout = WD_RECV_MAX_CNT_NOSLEEP; __u64 rx_cnt = 0; @@ -1824,7 +1835,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand if (balance) timeout = WD_RECV_MAX_CNT_SLEEP;
- ret = msg_handle->send(drv, ctx, msg); + ret = msg_handle->send(ctx, msg); if (unlikely(ret < 0)) { WD_ERR("failed to send msg to hw, ret = %d!\n", ret); return ret; @@ -1837,7 +1848,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand WD_ERR("wd ctx wait timeout(%d)!\n", ret); }
- ret = msg_handle->recv(drv, ctx, msg); + ret = msg_handle->recv(ctx, msg); if (ret != -WD_EAGAIN) { if (unlikely(ret < 0)) { WD_ERR("failed to recv msg: error = %d!\n", ret); @@ -1865,12 +1876,12 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) { if (!config || !config->ctxs || !config->ctxs[0].ctx) { - WD_ERR("invalid: config or config->ctxs is NULL!\n"); + WD_ERR("invalid: wd_ctx_config is NULL!\n"); return -WD_EINVAL; }
if (!sched) { - WD_ERR("invalid: sched is NULL!\n"); + WD_ERR("invalid: wd_sched is NULL!\n"); return -WD_EINVAL; }
@@ -1882,11 +1893,44 @@ int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) return 0; }
+int wd_alg_try_init(enum wd_status *status) +{ + enum wd_status expected; + __u32 count = 0; + bool ret; + + /* + * Here is aimed to protect the security of the initialization interface + * in the multi-thread scenario. Only one thread can get the WD_INITING + * status to initialize algorithm. Other thread will wait for the result. + * And the algorithm initialization interfaces is a liner process. + * So the initing thread will return a result to notify other thread go on. + */ + do { + expected = WD_UNINIT; + ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (expected == WD_INIT) { + WD_ERR("The algorithm has been initialized!\n"); + return -WD_EEXIST; + } + usleep(WD_INIT_SLEEP_UTIME); + + if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { + WD_ERR("The algorithm initialize wait timeout!\n"); + return -WD_ETIMEDOUT; + } + } while (!ret); + + return 0; +} + static void wd_get_alg_type(const char *alg_name, char *alg_type) { + int alg_nums = ARRAY_SIZE(alg_options); int i;
- for (i = 0; i < ARRAY_SIZE(alg_options); i++) { + for (i = 0; i < alg_nums; i++) { if (strcmp(alg_name, alg_options[i].name) == 0) { (void)strcpy(alg_type, alg_options[i].algtype); break; @@ -1946,10 +1990,21 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) }
int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) + struct wd_alg_driver *driver, void **drv_priv) { + void *priv; int ret;
+ if (!driver->priv_size) { + WD_ERR("invalid: driver priv ctx size is zero!\n"); + return -WD_EINVAL; + } + + /* Init ctx related resources in specific driver */ + priv = calloc(1, driver->priv_size); + if (!priv) + return -WD_ENOMEM; + if (!driver->init) { driver->fallback = 0; WD_ERR("driver have no init interface.\n"); @@ -1957,7 +2012,7 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, goto err_alloc; }
- ret = driver->init(driver, config); + ret = driver->init(config, priv); if (ret < 0) { WD_ERR("driver init failed.\n"); goto err_alloc; @@ -1970,23 +2025,31 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("soft alg driver init failed.\n"); } } + *drv_priv = priv;
return 0;
err_alloc: + free(priv); return ret; }
void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) + struct wd_alg_driver *driver, void **drv_priv) { + void *priv = *drv_priv;
- driver->exit(driver); + driver->exit(priv); /* Ctx config just need clear once */ wd_clear_ctx_config(config);
if (driver->fallback) wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); + + if (priv) { + free(priv); + *drv_priv = NULL; + } }
void wd_dlclose_drv(void *dlh_list) @@ -2200,11 +2263,11 @@ int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir)
if (is_dir) { len = snprintf(lib_path, PATH_MAX, "%s/%s", file_path, WD_DRV_LIB_DIR); - if (len >= PATH_MAX) + if (len < 0) return -WD_EINVAL; } else { len = snprintf(lib_path, PATH_MAX, "%s/%s/%s", file_path, WD_DRV_LIB_DIR, lib_file); - if (len >= PATH_MAX) + if (len < 0) return -WD_EINVAL; }
@@ -2350,31 +2413,6 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv) wd_release_drv(drv); }
-int wd_alg_try_init(enum wd_status *status) -{ - enum wd_status expected; - __u32 count = 0; - bool ret; - - do { - expected = WD_UNINIT; - ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); - if (expected == WD_INIT) { - WD_ERR("The algorithm has been initialized!\n"); - return -WD_EEXIST; - } - usleep(WD_INIT_SLEEP_UTIME); - - if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { - WD_ERR("The algorithm initialize wait timeout!\n"); - return -WD_ETIMEDOUT; - } - } while (!ret); - - return 0; -} - static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) { __u32 count = 0; @@ -2715,8 +2753,8 @@ free_ctxs:
static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) { - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); + free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); + free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); free(ctx_config->ctxs); }