Synchronize internal development codes to keep basic functional codes consistent.

Signed-off-by: Longfang Liu liulongfang@huawei.com --- drv/hash_mb/hash_mb.c | 83 +++++----- drv/hisi_comp.c | 45 ++--- drv/hisi_dae.c | 32 ++-- drv/hisi_hpre.c | 358 +++++++++++++++++++--------------------- drv/hisi_qm_udrv.c | 93 +++++------ drv/hisi_qm_udrv.h | 11 +- drv/hisi_sec.c | 150 +++++++++-------- drv/isa_ce_sm3.c | 37 ++--- drv/isa_ce_sm4.c | 41 ++--- include/wd.h | 21 +-- include/wd_aead.h | 1 + include/wd_alg.h | 34 +--- include/wd_alg_common.h | 28 ++-- include/wd_comp.h | 3 +- include/wd_digest.h | 9 +- include/wd_util.h | 27 +-- wd.c | 12 +- wd_alg.c | 2 +- wd_cipher.c | 50 ++++-- wd_comp.c | 34 ++-- wd_digest.c | 41 ++--- wd_sched.c | 3 +- wd_util.c | 140 ++++++++++------ 23 files changed, 625 insertions(+), 630 deletions(-)

diff --git a/drv/hash_mb/hash_mb.c b/drv/hash_mb/hash_mb.c index e4a9564..3f5b1a5 100644 --- a/drv/hash_mb/hash_mb.c +++ b/drv/hash_mb/hash_mb.c @@ -1,5 +1,15 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ +/* + * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. + */

#include <sys/auxv.h> #include <pthread.h> @@ -19,7 +29,6 @@ #define HASH_PADDING_BLOCKS 2 #define HASH_NENO_PROCESS_JOBS 4 #define HASH_TRY_PROCESS_COUNT 16 -#define BYTES_TO_BITS_OFFSET 3

#define MD5_DIGEST_DATA_SIZE 16 #define SM3_DIGEST_DATA_SIZE 32 @@ -109,7 +118,7 @@ static void hash_mb_queue_uninit(struct wd_ctx_config_internal *config, int ctx_ int i;

for (i = 0; i < ctx_num; i++) { - ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; + ctx = (struct wd_soft_ctx *)(uintptr_t)config->ctxs[i].ctx; mb_queue = ctx->priv; pthread_spin_destroy(&mb_queue->r_lock); hash_mb_uninit_poll_queue(&mb_queue->sm3_poll_queue); @@ -150,7 +159,7 @@ static int hash_mb_queue_init(struct wd_ctx_config_internal *config) }

mb_queue->ctx_mode = config->ctxs[i].ctx_mode; - ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; + ctx = (struct wd_soft_ctx *)(uintptr_t)config->ctxs[i].ctx; ctx->priv = mb_queue; ret = hash_mb_init_poll_queue(&mb_queue->sm3_poll_queue); if (ret) @@ -186,52 +195,41 @@ free_mb_queue: return ret; }

-static int hash_mb_init(struct wd_alg_driver *drv, void *conf) +static int hash_mb_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct hash_mb_ctx *priv; - int ret; + struct hash_mb_ctx *mb_ctx = priv;

/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;

- priv = malloc(sizeof(struct hash_mb_ctx)); - if (!priv) - return -WD_ENOMEM; - /* multibuff does not use epoll. */ config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - - ret = hash_mb_queue_init(config); - if (ret) { - free(priv); - return ret; - } + memcpy(&mb_ctx->config, config, sizeof(struct wd_ctx_config_internal));

- drv->priv = priv; - - return WD_SUCCESS; + return hash_mb_queue_init(config); }

-static void hash_mb_exit(struct wd_alg_driver *drv) +static void hash_mb_exit(void *priv) { - struct hash_mb_ctx *priv = (struct hash_mb_ctx *)drv->priv; + struct hash_mb_ctx *mb_ctx = priv; + struct wd_ctx_config_internal *config;

- if (!priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + }

- hash_mb_queue_uninit(&priv->config, priv->config.ctx_num); - free(priv); - drv->priv = NULL; + config = &mb_ctx->config; + hash_mb_queue_uninit(config, config->ctx_num); }

static void hash_mb_pad_data(struct hash_pad *hash_pad, __u8 *in, __u32 partial, __u64 total_len, bool transfer) { - __u64 size = total_len << BYTES_TO_BITS_OFFSET; __u8 *buffer = hash_pad->pad; + __u64 size = total_len << 3;

if (partial) memcpy(buffer, in, partial); @@ -266,7 +264,7 @@ static inline void hash_xor(__u8 *key_out, __u8 *key_in, __u32 key_len, __u8 xor if (i < key_len) key_out[i] = key_in[i] ^ xor_value; else - key_out[i] = xor_value; + key_out[i] = 0x0 ^ xor_value; } }

@@ -391,8 +389,7 @@ static int hash_first_block_process(struct wd_digest_msg *d_msg, memcpy(d_msg->partial_block, buffer, d_msg->partial_bytes); }

- /* - * Long hash mode, if first block is less than HASH_BLOCK_SIZE, + /* Long hash mode, if first block is less than HASH_BLOCK_SIZE, * copy ikey hash result to out. */ if (!job->len) { @@ -550,9 +547,9 @@ static int hash_mb_check_param(struct hash_mb_queue *mb_queue, struct wd_digest_ return WD_SUCCESS; }

-static int hash_mb_send(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static int hash_mb_send(handle_t ctx, void *drv_msg) { - struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; + struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)(uintptr_t)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; struct wd_digest_msg *d_msg = drv_msg; struct hash_mb_poll_queue *poll_queue; @@ -771,9 +768,9 @@ static int hash_mb_do_jobs(struct hash_mb_queue *mb_queue) return WD_SUCCESS; }

-static int hash_mb_recv(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static int hash_mb_recv(handle_t ctx, void *drv_msg) { - struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; + struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)(uintptr_t)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; struct wd_digest_msg *msg = drv_msg; int ret, i = 0; @@ -805,6 +802,7 @@ static int hash_mb_get_usage(void *param) .alg_name = (hash_alg_name),\ .calc_type = UADK_ALG_SVE_INSTR,\ .priority = 100,\ + .priv_size = sizeof(struct hash_mb_ctx),\ .queue_num = 1,\ .op_type_num = 1,\ .fallback = 0,\ @@ -822,14 +820,18 @@ static struct wd_alg_driver hash_mb_driver[] = {

static void __attribute__((constructor)) hash_mb_probe(void) { + unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); size_t i; int ret;

+ if (!(auxval & HWCAP_SVE)) + return; + WD_INFO("Info: register hash_mb alg drivers!\n"); for (i = 0; i < alg_num; i++) { ret = wd_alg_driver_register(&hash_mb_driver[i]); - if (ret && ret != -WD_ENODEV) + if (ret) WD_ERR("Error: register hash multibuff %s failed!\n", hash_mb_driver[i].alg_name); } @@ -837,11 +839,14 @@ static void __attribute__((constructor)) hash_mb_probe(void)

static void __attribute__((destructor)) hash_mb_remove(void) { + unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); size_t i;

+ if (!(auxval & HWCAP_SVE)) + return; + WD_INFO("Info: unregister hash_mb alg drivers!\n"); for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&hash_mb_driver[i]); } - diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 4c8e18b..ffe8eac 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -338,8 +338,7 @@ static void fill_buf_sgl_skip(struct hisi_zip_sqe *sqe, __u32 src_skip, }

static int fill_buf_deflate_slg_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg, const char *head, - int head_size) + struct wd_comp_msg *msg, const char *head, int head_size) { struct wd_comp_req *req = &msg->req; __u32 out_size = msg->avail_out; @@ -510,6 +509,8 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_type_sgl(sqe);

seq_start = get_seq_start_list(req); + if (unlikely(!seq_start)) + return -WD_EINVAL;

data->literals_start = req->list_dst; data->sequences_start = seq_start; @@ -700,7 +701,7 @@ static void get_data_size_lz77_zstd(struct hisi_zip_sqe *sqe, enum wd_comp_op_ty struct wd_lz77_zstd_data *data = recv_msg->req.priv; void *ctx_buf = recv_msg->ctx_buf;

- if (unlikely(!data)) + if (!data) return;

data->lit_num = sqe->comp_data_length; @@ -785,11 +786,11 @@ static void hisi_zip_sqe_ops_adapt(handle_t h_qp) } }

-static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) +static int hisi_zip_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_zip_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -799,11 +800,7 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }

- priv = malloc(sizeof(struct hisi_zip_ctx)); - if (!priv) - return -WD_EINVAL; - - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); + memcpy(&zip_ctx->config, config, sizeof(struct wd_ctx_config_internal)); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { h_ctx = config->ctxs[i].ctx; @@ -821,7 +818,6 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) }

hisi_zip_sqe_ops_adapt(h_qp); - drv->priv = priv;

return 0; out: @@ -829,29 +825,20 @@ out: h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); hisi_qm_free_qp(h_qp); } - free(priv); return -WD_EINVAL; }

-static void hisi_zip_exit(struct wd_alg_driver *drv) +static void hisi_zip_exit(void *priv) { - struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; - struct wd_ctx_config_internal *config; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; + struct wd_ctx_config_internal *config = &zip_ctx->config; handle_t h_qp; __u32 i;

- if (!priv) { - /* return if already exit */ - return; - } - - config = &priv->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; }

static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, @@ -931,7 +918,7 @@ static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, } }

-static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_send(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *msg = comp_msg; @@ -1001,14 +988,13 @@ static void get_ctx_buf(struct hisi_zip_sqe *sqe, }

static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg) + struct wd_comp_msg *recv_msg) { __u32 buf_type = (sqe->dw9 & HZ_BUF_TYPE_MASK) >> BUF_TYPE_SHIFT; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; __u32 status = sqe->dw3 & HZ_STATUS_MASK; __u32 type = sqe->dw9 & HZ_REQ_TYPE_MASK; - struct wd_comp_msg *recv_msg = msg; bool need_debug = wd_need_debug(); int alg_type, ret; __u32 tag; @@ -1045,7 +1031,6 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, }

ops[alg_type].get_data_size(sqe, qp->q_info.qc_type, recv_msg); - get_ctx_buf(sqe, recv_msg);

/* last block no space, need resend null size req */ @@ -1054,7 +1039,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe,

if (need_debug) WD_DEBUG("zip recv lst =%hu, ctx_st=0x%x, status=0x%x, alg=%u!\n", - lstblk, ctx_st, status, type); + lstblk, ctx_st, status, type); if (lstblk && (status == HZ_DECOMP_END)) recv_msg->req.status = WD_STREAM_END;

@@ -1071,7 +1056,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, return 0; }

-static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_recv(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *recv_msg = comp_msg; @@ -1093,6 +1078,7 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com .alg_name = (zip_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_zip_ctx),\ .queue_num = ZIP_CTX_Q_NUM_DEF,\ .op_type_num = 2,\ .fallback = 0,\ @@ -1105,7 +1091,6 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com static struct wd_alg_driver zip_alg_driver[] = { GEN_ZIP_ALG_DRIVER("zlib"), GEN_ZIP_ALG_DRIVER("gzip"), - GEN_ZIP_ALG_DRIVER("deflate"), GEN_ZIP_ALG_DRIVER("lz77_zstd"), }; diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index 09d2387..70d01c9 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -518,7 +518,7 @@ static int check_hashagg_param(struct wd_agg_msg *msg) return WD_SUCCESS; }

-static int hashagg_send(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_send(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -651,7 +651,7 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms } }

-static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_recv(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1546,11 +1546,11 @@ update_table: return ret; }

-static int dae_init(struct wd_alg_driver *drv, void *conf) +static int dae_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_dae_ctx *dae_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_dae_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -1561,10 +1561,6 @@ static int dae_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }

- priv = malloc(sizeof(struct hisi_dae_ctx)); - if (!priv) - return -WD_ENOMEM; - qm_priv.op_type = DAE_HASH_AGG_TYPE; qm_priv.sqe_size = sizeof(struct dae_sqe); /* Allocate qp for each context */ @@ -1585,10 +1581,9 @@ static int dae_init(struct wd_alg_driver *drv, void *conf) if (ret) goto free_h_qp; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&dae_ctx->config, config, sizeof(struct wd_ctx_config_internal));

- return WD_SUCCESS; + return 0;

free_h_qp: hisi_qm_free_qp(h_qp); @@ -1598,29 +1593,27 @@ out: dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - free(priv); return ret; }

-static void dae_exit(struct wd_alg_driver *drv) +static void dae_exit(void *priv) { - struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; + struct hisi_dae_ctx *dae_ctx = priv; struct wd_ctx_config_internal *config; handle_t h_qp; __u32 i;

- if (!priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + }

- config = &priv->config; + config = &dae_ctx->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - - free(priv); - drv->priv = NULL; }

static int dae_get_usage(void *param) @@ -1648,6 +1641,7 @@ static struct wd_alg_driver hashagg_driver = { .alg_name = "hashagg", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_dae_ctx), .queue_num = DAE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 37bb5ee..1f02701 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -1,4 +1,3 @@ - /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */

@@ -40,31 +39,31 @@ #define GEN_PARAMS_SZ(key_size) ((key_size) << 1) #define CRT_PARAM_SZ(key_size) ((key_size) >> 1)

-#define WD_TRANS_FAIL 0 +#define WD_TRANS_FAIL 0

enum hpre_alg_type { - HPRE_ALG_NC_NCRT = 0x0, - HPRE_ALG_NC_CRT = 0x1, - HPRE_ALG_KG_STD = 0x2, - HPRE_ALG_KG_CRT = 0x3, - HPRE_ALG_DH_G2 = 0x4, - HPRE_ALG_DH = 0x5, - HPRE_ALG_PRIME = 0x6, - HPRE_ALG_MOD = 0x7, - HPRE_ALG_MOD_INV = 0x8, - HPRE_ALG_MUL = 0x9, - HPRE_ALG_COPRIME = 0xA, - HPRE_ALG_ECC_CURVE_TEST = 0xB, - HPRE_ALG_ECDH_PLUS = 0xC, - HPRE_ALG_ECDH_MULTIPLY = 0xD, - HPRE_ALG_ECDSA_SIGN = 0xE, - HPRE_ALG_ECDSA_VERF = 0xF, - HPRE_ALG_X_DH_MULTIPLY = 0x10, - HPRE_ALG_SM2_KEY_GEN = 0x11, - HPRE_ALG_SM2_SIGN = 0x12, - HPRE_ALG_SM2_VERF = 0x13, - HPRE_ALG_SM2_ENC = 0x14, - HPRE_ALG_SM2_DEC = 0x15 + HW_ALG_NC_NCRT = 0x0, + HW_ALG_NC_CRT = 0x1, + HW_ALG_KG_STD = 0x2, + HW_ALG_KG_CRT = 0x3, + HW_ALG_DH_G2 = 0x4, + HW_ALG_DH = 0x5, + HW_ALG_PRIME = 0x6, + HW_ALG_MOD = 0x7, + HW_ALG_MOD_INV = 0x8, + HW_ALG_MUL = 0x9, + HW_ALG_COPRIME = 0xA, + HW_ALG_ECC_CURVE_TEST = 0xB, + HW_ALG_ECDH_PLUS = 0xC, + HW_ALG_ECDH_MULTIPLY = 0xD, + HW_ALG_ECDSA_SIGN = 0xE, + HW_ALG_ECDSA_VERF = 0xF, + HW_ALG_X_DH_MULTIPLY = 0x10, + HW_ALG_SM2_KEY_GEN = 0x11, + HW_ALG_SM2_SIGN = 0x12, + HW_ALG_SM2_VERF = 0x13, + HW_ALG_SM2_ENC = 0x14, + HW_ALG_SM2_DEC = 0x15 };

enum hpre_alg_name { @@ -141,7 +140,7 @@ static void dump_hpre_msg(void *msg, int alg) } }

-static bool is_hpre_bin_fmt(char *dst, const char *src, __u32 dsz, __u32 bsz) +static bool is_hpre_bin_fmt(char *dst, const char *src, int dsz, int bsz) { const char *temp = src + dsz; int lens = bsz - dsz; @@ -227,17 +226,11 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey,

wd_rsa_get_crt_prikey_params(prikey, &wd_dq, &wd_dp, &wd_qinv, &wd_q, &wd_p); - ret = crypto_bin_to_hpre_bin(wd_dq->data, (const char *)wd_dq->data, - wd_dq->bsize, wd_dq->dsize, "rsa crt dq"); - if (ret) - return ret; - wd_dq->dsize = wd_dq->bsize; - - ret = crypto_bin_to_hpre_bin(wd_dp->data, (const char *)wd_dp->data, - wd_dp->bsize, wd_dp->dsize, "rsa crt dp"); + ret = crypto_bin_to_hpre_bin(wd_p->data, + (const char *)wd_p->data, wd_p->bsize, wd_p->dsize, "rsa crt p"); if (ret) return ret; - wd_dp->dsize = wd_dp->bsize; + wd_p->dsize = wd_p->bsize;

ret = crypto_bin_to_hpre_bin(wd_q->data, (const char *)wd_q->data, wd_q->bsize, wd_q->dsize, "rsa crt q"); @@ -245,12 +238,6 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey, return ret; wd_q->dsize = wd_q->bsize;

- ret = crypto_bin_to_hpre_bin(wd_p->data, - (const char *)wd_p->data, wd_p->bsize, wd_p->dsize, "rsa crt p"); - if (ret) - return ret; - wd_p->dsize = wd_p->bsize; - ret = crypto_bin_to_hpre_bin(wd_qinv->data, (const char *)wd_qinv->data, wd_qinv->bsize, wd_qinv->dsize, "rsa crt qinv"); @@ -258,6 +245,18 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey, return ret; wd_qinv->dsize = wd_qinv->bsize;

+ ret = crypto_bin_to_hpre_bin(wd_dq->data, (const char *)wd_dq->data, + wd_dq->bsize, wd_dq->dsize, "rsa crt dq"); + if (ret) + return ret; + wd_dq->dsize = wd_dq->bsize; + + ret = crypto_bin_to_hpre_bin(wd_dp->data, (const char *)wd_dp->data, + wd_dp->bsize, wd_dp->dsize, "rsa crt dp"); + if (ret) + return ret; + wd_dp->dsize = wd_dp->bsize; + *data = wd_dq->data;

return WD_SUCCESS; @@ -269,18 +268,18 @@ static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data) int ret;

wd_rsa_get_prikey_params(prikey, &wd_d, &wd_n); - ret = crypto_bin_to_hpre_bin(wd_d->data, (const char *)wd_d->data, - wd_d->bsize, wd_d->dsize, "rsa d"); - if (ret) - return ret; - wd_d->dsize = wd_d->bsize; - ret = crypto_bin_to_hpre_bin(wd_n->data, (const char *)wd_n->data, wd_n->bsize, wd_n->dsize, "rsa n"); if (ret) return ret; wd_n->dsize = wd_n->bsize;

+ ret = crypto_bin_to_hpre_bin(wd_d->data, (const char *)wd_d->data, + wd_d->bsize, wd_d->dsize, "rsa d"); + if (ret) + return ret; + wd_d->dsize = wd_d->bsize; + *data = wd_d->data;

return WD_SUCCESS; @@ -292,20 +291,19 @@ static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data) int ret;

wd_rsa_get_pubkey_params(pubkey, &wd_e, &wd_n); - ret = crypto_bin_to_hpre_bin(wd_e->data, (const char *)wd_e->data, - wd_e->bsize, wd_e->dsize, "rsa e"); - if (ret) - return ret; - wd_e->dsize = wd_e->bsize; - ret = crypto_bin_to_hpre_bin(wd_n->data, (const char *)wd_n->data, wd_n->bsize, wd_n->dsize, "rsa n"); if (ret) return ret; wd_n->dsize = wd_n->bsize;

- *data = wd_e->data; + ret = crypto_bin_to_hpre_bin(wd_e->data, (const char *)wd_e->data, + wd_e->bsize, wd_e->dsize, "rsa e"); + if (ret) + return ret; + wd_e->dsize = wd_e->bsize;

+ *data = wd_e->data; return WD_SUCCESS; }

@@ -315,19 +313,18 @@ static int fill_rsa_genkey_in(struct wd_rsa_kg_in *genkey) int ret;

wd_rsa_get_kg_in_params(genkey, &e, &q, &p); + ret = crypto_bin_to_hpre_bin(p.data, (const char *)p.data, + p.bsize, p.dsize, "rsa kg p"); + if (ret) + return ret;

ret = crypto_bin_to_hpre_bin(e.data, (const char *)e.data, e.bsize, e.dsize, "rsa kg e"); if (ret) return ret;

- ret = crypto_bin_to_hpre_bin(q.data, (const char *)q.data, + return crypto_bin_to_hpre_bin(q.data, (const char *)q.data, q.bsize, q.dsize, "rsa kg q"); - if (ret) - return ret; - - return crypto_bin_to_hpre_bin(p.data, (const char *)p.data, - p.bsize, p.dsize, "rsa kg p"); }

static int hpre_tri_bin_transfer(struct wd_dtb *bin0, struct wd_dtb *bin1, @@ -365,7 +362,7 @@ static int hpre_tri_bin_transfer(struct wd_dtb *bin0, struct wd_dtb *bin1, }

static int rsa_out_transfer(struct wd_rsa_msg *msg, - struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) + struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) { struct wd_rsa_req *req = &msg->req; struct wd_rsa_kg_out *key = req->dst; @@ -375,8 +372,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, void *data; int ret;

- if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) { - /* async */ + if (hw_msg->alg == HW_ALG_KG_CRT || hw_msg->alg == HW_ALG_KG_STD) { if (qp_mode == CTX_MODE_ASYNC) { data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); key = container_of(data, struct wd_rsa_kg_out, data); @@ -386,7 +382,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, }

msg->result = WD_SUCCESS; - if (hw_msg->alg == HPRE_ALG_KG_CRT) { + if (hw_msg->alg == HW_ALG_KG_CRT) { req->dst_bytes = CRT_GEN_PARAMS_SZ(kbytes); wd_rsa_get_kg_out_crt_params(key, &qinv, &dq, &dp); ret = hpre_tri_bin_transfer(&qinv, &dq, &dp); @@ -397,7 +393,7 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,

wd_rsa_set_kg_out_crt_psz(key, qinv.dsize, dq.dsize, dp.dsize); - } else if (hw_msg->alg == HPRE_ALG_KG_STD) { + } else if (hw_msg->alg == HW_ALG_KG_STD) { req->dst_bytes = GEN_PARAMS_SZ(kbytes);

wd_rsa_get_kg_out_params(key, &d, &n); @@ -423,7 +419,7 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, int ret;

if (req->op_type == WD_RSA_SIGN) { - if (hw_msg->alg == HPRE_ALG_NC_CRT) { + if (hw_msg->alg == HW_ALG_NC_CRT) { ret = fill_rsa_crt_prikey2((void *)msg->key, &data); if (ret) return ret; @@ -431,13 +427,13 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, ret = fill_rsa_prikey1((void *)msg->key, &data); if (ret) return ret; - hw_msg->alg = HPRE_ALG_NC_NCRT; + hw_msg->alg = HW_ALG_NC_NCRT; } } else if (req->op_type == WD_RSA_VERIFY) { ret = fill_rsa_pubkey((void *)msg->key, &data); if (ret) return ret; - hw_msg->alg = HPRE_ALG_NC_NCRT; + hw_msg->alg = HW_ALG_NC_NCRT; } else if (req->op_type == WD_RSA_GENKEY) { ret = fill_rsa_genkey_in((void *)msg->key); if (ret) @@ -447,10 +443,10 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, WD_ERR("failed to get rsa gen key data!\n"); return ret; } - if (hw_msg->alg == HPRE_ALG_NC_CRT) - hw_msg->alg = HPRE_ALG_KG_CRT; + if (hw_msg->alg == HW_ALG_NC_CRT) + hw_msg->alg = HW_ALG_KG_CRT; else - hw_msg->alg = HPRE_ALG_KG_STD; + hw_msg->alg = HW_ALG_KG_STD; } else { WD_ERR("invalid: rsa operatin type %u is error!\n", req->op_type); return -WD_EINVAL; @@ -525,11 +521,11 @@ out: return -WD_EINVAL; }

-static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) +static int hpre_rsa_dh_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret;

if (!config->ctx_num) { @@ -537,27 +533,19 @@ static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }

- priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V2_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv;

return WD_SUCCESS; }

-static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) +static int hpre_ecc_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret;

if (!config->ctx_num) { @@ -565,45 +553,28 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }

- priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V3_ECC_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv;

return WD_SUCCESS; }

-static void hpre_exit(struct wd_alg_driver *drv) +static void hpre_exit(void *priv) { - struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; - struct wd_ctx_config_internal *config; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; + struct wd_ctx_config_internal *config = &hpre_ctx->config; handle_t h_qp; __u32 i;

- if (!priv) { - /* return if already exit */ - return; - } - - config = &priv->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - - free(priv); - drv->priv = NULL; }

-static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_send(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_rsa_msg *msg = rsa_msg; @@ -615,9 +586,9 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)

if (msg->key_type == WD_RSA_PRIKEY1 || msg->key_type == WD_RSA_PUBKEY) - hw_msg.alg = HPRE_ALG_NC_NCRT; + hw_msg.alg = HW_ALG_NC_NCRT; else if (msg->key_type == WD_RSA_PRIKEY2) - hw_msg.alg = HPRE_ALG_NC_CRT; + hw_msg.alg = HW_ALG_NC_CRT; else return -WD_EINVAL;

@@ -659,14 +630,15 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, } }

-static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_recv(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); - struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct hisi_hpre_sqe hw_msg = {0}; struct wd_rsa_msg *msg = rsa_msg; struct wd_rsa_msg *temp_msg; + struct hisi_qp *qp; __u16 recv_cnt = 0; + __u64 tag; int ret;

ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt); @@ -677,12 +649,14 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) if (ret) return ret;

- msg->tag = LW_U16(hw_msg.low_tag); + qp = (struct hisi_qp *)h_qp; + tag = LW_U16(hw_msg.low_tag); + msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_rsa_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -741,7 +715,6 @@ static int dh_out_transfer(struct wd_dh_msg *msg, void *out; int ret;

- /* async */ if (qp_mode == CTX_MODE_ASYNC) out = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); else @@ -757,7 +730,7 @@ static int dh_out_transfer(struct wd_dh_msg *msg, return WD_SUCCESS; }

-static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_send(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_dh_msg *msg = dh_msg; @@ -769,9 +742,9 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));

if (msg->is_g2 && req->op_type != WD_DH_PHASE2) - hw_msg.alg = HPRE_ALG_DH_G2; + hw_msg.alg = HW_ALG_DH_G2; else - hw_msg.alg = HPRE_ALG_DH; + hw_msg.alg = HW_ALG_DH;

hw_msg.task_len1 = msg->key_bytes / BYTE_BITS - 0x1;

@@ -802,14 +775,15 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); }

-static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_recv(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); - struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_dh_msg *msg = dh_msg; struct hisi_hpre_sqe hw_msg = {0}; struct wd_dh_msg *temp_msg; + struct hisi_qp *qp; __u16 recv_cnt = 0; + __u64 tag; int ret;

ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt); @@ -820,12 +794,14 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) if (ret) return ret;

- msg->tag = LW_U16(hw_msg.low_tag); + qp = (struct hisi_qp *)h_qp; + tag = LW_U16(hw_msg.low_tag); + msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_dh_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -852,34 +828,34 @@ static int ecc_prepare_alg(struct wd_ecc_msg *msg, { switch (msg->req.op_type) { case WD_SM2_SIGN: - hw_msg->alg = HPRE_ALG_SM2_SIGN; + hw_msg->alg = HW_ALG_SM2_SIGN; break; case WD_ECDSA_SIGN: - hw_msg->alg = HPRE_ALG_ECDSA_SIGN; + hw_msg->alg = HW_ALG_ECDSA_SIGN; break; case WD_SM2_VERIFY: - hw_msg->alg = HPRE_ALG_SM2_VERF; + hw_msg->alg = HW_ALG_SM2_VERF; break; case WD_ECDSA_VERIFY: - hw_msg->alg = HPRE_ALG_ECDSA_VERF; + hw_msg->alg = HW_ALG_ECDSA_VERF; break; case WD_SM2_ENCRYPT: - hw_msg->alg = HPRE_ALG_SM2_ENC; + hw_msg->alg = HW_ALG_SM2_ENC; break; case WD_SM2_DECRYPT: - hw_msg->alg = HPRE_ALG_SM2_DEC; + hw_msg->alg = HW_ALG_SM2_DEC; break; case WD_SM2_KG: - hw_msg->alg = HPRE_ALG_SM2_KEY_GEN; + hw_msg->alg = HW_ALG_SM2_KEY_GEN; break; case WD_ECXDH_GEN_KEY: /* fall through */ case HPRE_SM2_ENC: /* fall through */ case HPRE_SM2_DEC: /* fall through */ case WD_ECXDH_COMPUTE_KEY: if (msg->curve_id == WD_X448 || msg->curve_id == WD_X25519) - hw_msg->alg = HPRE_ALG_X_DH_MULTIPLY; + hw_msg->alg = HW_ALG_X_DH_MULTIPLY; else - hw_msg->alg = HPRE_ALG_ECDH_MULTIPLY; + hw_msg->alg = HW_ALG_ECDH_MULTIPLY; break; default: return -WD_EINVAL; @@ -957,8 +933,8 @@ static bool less_than_latter(struct wd_dtb *d, struct wd_dtb *n) ret = memcmp(d->data + shift, n->data + shift, n->dsize); if (ret < 0) return true; - else - return false; + + return false; }

static int ecc_prepare_prikey(struct wd_ecc_key *key, void **data, int id) @@ -1100,16 +1076,16 @@ static int ecc_prepare_key(struct wd_ecc_msg *msg, static void ecc_get_io_len(__u32 atype, __u32 hsz, size_t *ilen, size_t *olen) { - if (atype == HPRE_ALG_ECDH_MULTIPLY) { + if (atype == HW_ALG_ECDH_MULTIPLY) { *olen = ECDH_OUT_PARAMS_SZ(hsz); *ilen = *olen; - } else if (atype == HPRE_ALG_X_DH_MULTIPLY) { + } else if (atype == HW_ALG_X_DH_MULTIPLY) { *olen = X_DH_OUT_PARAMS_SZ(hsz); *ilen = *olen; - } else if (atype == HPRE_ALG_ECDSA_SIGN) { + } else if (atype == HW_ALG_ECDSA_SIGN) { *olen = ECC_SIGN_OUT_PARAMS_SZ(hsz); *ilen = ECC_SIGN_IN_PARAMS_SZ(hsz); - } else if (atype == HPRE_ALG_ECDSA_VERF) { + } else if (atype == HW_ALG_ECDSA_VERF) { *olen = ECC_VERF_OUT_PARAMS_SZ; *ilen = ECC_VERF_IN_PARAMS_SZ(hsz); } else { @@ -1561,31 +1537,31 @@ static int set_prikey(struct wd_ecc_prikey *prikey, struct wd_sm2_enc_in *ein = msg->req.src; int ret;

- ret = set_param(&prikey->p, &pubkey->p, "p"); + ret = set_param(&prikey->g.x, &pubkey->g.x, "gx"); if (unlikely(ret)) return ret;

- ret = set_param(&prikey->a, &pubkey->a, "a"); + ret = set_param(&prikey->b, &pubkey->b, "b"); if (unlikely(ret)) return ret;

- ret = set_param(&prikey->d, &ein->k, "k"); + ret = set_param(&prikey->a, &pubkey->a, "a"); if (unlikely(ret)) return ret;

- ret = set_param(&prikey->b, &pubkey->b, "b"); + ret = set_param(&prikey->d, &ein->k, "k"); if (unlikely(ret)) return ret;

- ret = set_param(&prikey->n, &pubkey->n, "n"); - if (unlikely(ret)) + ret = set_param(&prikey->g.y, &pubkey->g.y, "gy"); + if (ret) return ret;

- ret = set_param(&prikey->g.x, &pubkey->g.x, "gx"); + ret = set_param(&prikey->p, &pubkey->p, "p"); if (unlikely(ret)) return ret;

- return set_param(&prikey->g.y, &pubkey->g.y, "gy"); + return set_param(&prikey->n, &pubkey->n, "n"); }

static struct wd_ecc_out *create_ecdh_out(struct wd_ecc_msg *msg) @@ -1759,7 +1735,6 @@ static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) hw_msg->etype = 0x0; hw_msg->low_tag = msg->tag; hw_msg->task_len1 = hw_sz / BYTE_BITS - 0x1; - return ret; }

@@ -1893,7 +1868,7 @@ free_dst: return ret; }

-static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_send(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -1920,7 +1895,7 @@ static int ecdh_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_ms

wd_ecxdh_get_out_params(out, &key);

- if (hw_msg->alg == HPRE_ALG_ECDH_MULTIPLY) + if (hw_msg->alg == HW_ALG_ECDH_MULTIPLY) y = &key->y;

ret = hpre_tri_bin_transfer(&key->x, y, NULL); @@ -2012,26 +1987,25 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, int ret = -WD_EINVAL; void *va;

- /* async */ if (qp_mode == CTX_MODE_ASYNC) { va = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); msg->req.dst = container_of(va, struct wd_ecc_out, data); }

- if (hw_msg->alg == HPRE_ALG_SM2_SIGN || - hw_msg->alg == HPRE_ALG_ECDSA_SIGN) + if (hw_msg->alg == HW_ALG_SM2_SIGN || + hw_msg->alg == HW_ALG_ECDSA_SIGN) ret = ecc_sign_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_VERF || - hw_msg->alg == HPRE_ALG_ECDSA_VERF) + else if (hw_msg->alg == HW_ALG_SM2_VERF || + hw_msg->alg == HW_ALG_ECDSA_VERF) ret = ecc_verf_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_ENC) + else if (hw_msg->alg == HW_ALG_SM2_ENC) ret = sm2_enc_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_SM2_DEC) + else if (hw_msg->alg == HW_ALG_SM2_DEC) ret = 0; - else if (hw_msg->alg == HPRE_ALG_SM2_KEY_GEN) + else if (hw_msg->alg == HW_ALG_SM2_KEY_GEN) ret = sm2_kg_out_transfer(msg, hw_msg); - else if (hw_msg->alg == HPRE_ALG_ECDH_MULTIPLY || - hw_msg->alg == HPRE_ALG_X_DH_MULTIPLY) + else if (hw_msg->alg == HW_ALG_ECDH_MULTIPLY || + hw_msg->alg == HW_ALG_X_DH_MULTIPLY) ret = ecdh_out_transfer(msg, hw_msg); else WD_ERR("invalid: algorithm type %u is error!\n", hw_msg->alg); @@ -2039,6 +2013,15 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, return ret; }

+static void msg_pack(char *dst, __u64 *out_len, + const void *src, __u32 src_len) +{ + if (unlikely(!src || !src_len)) + return; + + memcpy(dst + *out_len, src, src_len); + *out_len += src_len; +}

static __u32 get_hash_bytes(__u8 type) { @@ -2073,14 +2056,13 @@ static __u32 get_hash_bytes(__u8 type) return val; }

-static void msg_pack(char *dst, __u64 *out_len, - const void *src, __u32 src_len) +static void sm2_xor(struct wd_dtb *val1, struct wd_dtb *val2) { - if (unlikely(!src || !src_len)) - return; + __u32 i;

- memcpy(dst + *out_len, src, src_len); - *out_len += src_len; + for (i = 0; i < val1->dsize; ++i) + val1->data[i] = (char)((__u8)val1->data[i] ^ + (__u8)val2->data[i]); }

static int sm2_kdf(struct wd_dtb *out, struct wd_ecc_point *x2y2, @@ -2145,15 +2127,6 @@ static int sm2_kdf(struct wd_dtb *out, struct wd_ecc_point *x2y2, return ret; }

-static void sm2_xor(struct wd_dtb *val1, struct wd_dtb *val2) -{ - __u32 i; - - for (i = 0; i < val1->dsize; ++i) - val1->data[i] = (char)((__u8)val1->data[i] ^ - (__u8)val2->data[i]); -} - static int is_equal(struct wd_dtb *src, struct wd_dtb *dst) { if (src->dsize == dst->dsize && @@ -2309,14 +2282,17 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { struct wd_ecc_msg *temp_msg; + __u64 tag; int ret;

- msg->tag = LW_U16(hw_msg->low_tag); + tag = LW_U16(hw_msg->low_tag); + msg->tag = tag; + if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_ecc_get_msg(qp->q_info.idx, tag); if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); + WD_ERR("failed to get send msg! idx = %u, tag = %llu.\n", + qp->q_info.idx, tag); return -WD_EINVAL; } } else { @@ -2326,19 +2302,19 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, hpre_result_check(hw_msg, &msg->result); if (msg->result) { ret = -msg->result; - goto dump_err_msg; + goto err; }

ret = ecc_out_transfer(msg, hw_msg, qp->q_info.qp_mode); if (ret) { msg->result = WD_OUT_EPARA; WD_ERR("failed to transfer out ecc BD, ret = %d!\n", ret); - goto dump_err_msg; + goto err; }

return ret;

-dump_err_msg: +err: dump_hpre_msg(temp_msg, WD_ECC);

return ret; @@ -2373,6 +2349,7 @@ static int parse_second_sqe(handle_t h_qp, hsz = (hw_msg.task_len1 + 1) * BYTE_BITS; dst = *(struct wd_ecc_msg **)((uintptr_t)data + hsz * ECDH_OUT_PARAM_NUM); + ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg); msg->result = dst->result; *second = dst; @@ -2380,8 +2357,8 @@ static int parse_second_sqe(handle_t h_qp, return ret; }

-static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_enc_parse(handle_t h_qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *second = NULL; @@ -2464,7 +2441,7 @@ fail: return ret; }

-static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_recv(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2480,10 +2457,10 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) if (ret) return ret;

- if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && + if (hw_msg.alg == HW_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_ENC) return sm2_enc_parse(h_qp, msg, &hw_msg); - else if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && + else if (hw_msg.alg == HW_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_DEC) return sm2_dec_parse(h_qp, msg, &hw_msg);

@@ -2501,6 +2478,7 @@ static int hpre_get_usage(void *param) .alg_name = (hpre_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_hpre_ctx),\ .queue_num = HPRE_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ .fallback = 0,\ @@ -2524,6 +2502,7 @@ static struct wd_alg_driver hpre_rsa_driver = { .alg_name = "rsa", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -2539,6 +2518,7 @@ static struct wd_alg_driver hpre_dh_driver = { .alg_name = "dh", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -2566,7 +2546,7 @@ static void __attribute__((constructor)) hisi_hpre_probe(void)

ret = wd_alg_driver_register(&hpre_dh_driver); if (ret && ret != -WD_ENODEV) - WD_ERR("failed to register HPRE dh driver!\n"); + WD_ERR("failed to register HPRE rsa driver!\n");

for (i = 0; i < alg_num; i++) { ret = wd_alg_driver_register(&hpre_ecc_driver[i]); diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 304764e..30d20ff 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -8,7 +8,6 @@ #include <sys/mman.h>

#include "hisi_qm_udrv.h" -#include "wd_util.h"

#define QM_DBELL_CMD_SQ 0 #define QM_DBELL_CMD_CQ 1 @@ -25,9 +24,9 @@ #define CQE_SQ_HEAD_INDEX(cqe) (__le16_to_cpu((cqe)->sq_head) & 0xffff) #define VERSION_ID_SHIFT 9

-#define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) +#define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) #define UACCE_CMD_QM_SET_QP_INFO _IOWR('H', 11, struct hisi_qp_info) -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))

/* the max sge num in one sgl */ #define HISI_SGE_NUM_IN_SGL 255 @@ -354,12 +353,12 @@ static int hisi_qm_setup_info(struct hisi_qp *qp, struct hisi_qm_priv *config) ret = pthread_spin_init(&q_info->sd_lock, PTHREAD_PROCESS_SHARED); if (ret) { WD_DEV_ERR(qp->h_ctx, "failed to init qinfo sd_lock!\n"); - goto err_destroy_lock; + goto err_destory_lock; }

return 0;

-err_destroy_lock: +err_destory_lock: pthread_spin_destroy(&q_info->rv_lock); err_out: hisi_qm_unset_region(qp->h_ctx, q_info); @@ -395,22 +394,22 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx) int ret;

if (!config) - goto out; + return (handle_t)NULL;

if (!config->sqe_size) { - WD_ERR("invalid: sqe size is zero!\n"); - goto out; + WD_DEV_ERR(ctx, "invalid: sqe size is zero!\n"); + return (handle_t)NULL; }

qp = calloc(1, sizeof(struct hisi_qp)); if (!qp) - goto out; + return (handle_t)NULL;

qp->h_ctx = ctx;

ret = hisi_qm_setup_info(qp, config); if (ret) - goto out_qp; + goto free_qp;

qp->h_sgl_pool = hisi_qm_create_sglpool(HISI_SGL_NUM_IN_BD, HISI_SGE_NUM_IN_SGL); @@ -433,9 +432,8 @@ free_pool: hisi_qm_destroy_sglpool(qp->h_sgl_pool); free_info: hisi_qm_clear_info(qp); -out_qp: +free_qp: free(qp); -out: return (handle_t)NULL; }

@@ -562,46 +560,14 @@ static int hisi_qm_recv_single(struct hisi_qm_queue_info *q_info, handle_t h_ctx return 0; }

-int hisi_qm_recv(handle_t h_qp, void *resp, __u16 expect, __u16 *count) -{ - struct hisi_qp *qp = (struct hisi_qp *)h_qp; - struct hisi_qm_queue_info *q_info; - __u16 recv_num = 0; - __u16 i; - int ret; - - if (unlikely(!resp || !qp || !count)) - return -WD_EINVAL; - - if (unlikely(!expect)) - return 0; - - q_info = &qp->q_info; - if (unlikely(wd_ioread32(q_info->ds_rx_base) == 1)) { - WD_DEV_ERR(qp->h_ctx, "wd queue hw error happened before qm receive!\n"); - return -WD_HW_EACCESS; - } - - for (i = 0; i < expect; i++) { - ret = hisi_qm_recv_single(q_info, qp->h_ctx, resp, i); - if (ret) - break; - recv_num++; - } - - *count = recv_num; - - return ret; -} - int hisi_check_bd_id(handle_t h_qp, __u32 mid, __u32 bid) { struct hisi_qp *qp = (struct hisi_qp *)h_qp; __u8 mode = qp->q_info.qp_mode;

if (mode == CTX_MODE_SYNC && mid != bid) { - WD_DEV_ERR(qp->h_ctx, "failed to recv self bd, send id: %u, recv id: %u\n", - mid, bid); + WD_DEV_ERR(qp->h_ctx, "failed to recv bd, send id:%u, recv id:%u!\n", + mid, bid); return -WD_EINVAL; }

@@ -630,6 +596,39 @@ void hisi_set_msg_id(handle_t h_qp, __u32 *tag) } }

+int hisi_qm_recv(handle_t h_qp, void *resp, __u16 expect, __u16 *count) +{ + struct hisi_qp *qp = (struct hisi_qp *)h_qp; + struct hisi_qm_queue_info *q_info; + __u16 recv_num = 0; + __u16 i; + int ret; + + if (unlikely(!resp || !qp || !count)) + return -WD_EINVAL; + + if (unlikely(!expect)) + return 0; + + q_info = &qp->q_info; + + if (unlikely(wd_ioread32(q_info->ds_rx_base) == 1)) { + WD_DEV_ERR(qp->h_ctx, "wd queue hw error happened before qm receive!\n"); + return -WD_HW_EACCESS; + } + + for (i = 0; i < expect; i++) { + ret = hisi_qm_recv_single(q_info, qp->h_ctx, resp, i); + if (ret) + break; + recv_num++; + } + + *count = recv_num; + + return ret; +} + static void *hisi_qm_create_sgl(__u32 sge_num) { void *sgl; @@ -745,7 +744,9 @@ void hisi_qm_destroy_sglpool(handle_t sgl_pool) }

pool = (struct hisi_sgl_pool *)sgl_pool; + pthread_spin_destroy(&pool->lock); + hisi_qm_free_sglpool(pool); }

diff --git a/drv/hisi_qm_udrv.h b/drv/hisi_qm_udrv.h index b02e8e7..0220e89 100644 --- a/drv/hisi_qm_udrv.h +++ b/drv/hisi_qm_udrv.h @@ -6,11 +6,13 @@

#include <stdbool.h> #include <stddef.h> +#include <stdint.h> #include <pthread.h> #include <linux/types.h>

-#include "config.h" -#include "wd_util.h" +#include "uacce.h" +#include "wd.h" +#include "wd_alg_common.h"

#ifdef __cplusplus extern "C" { @@ -86,7 +88,6 @@ struct hisi_qp { struct hisi_qm_queue_info q_info; handle_t h_sgl_pool; handle_t h_ctx; - /* Private area for driver use, point to queue specifial data */ void *priv; };

@@ -126,7 +127,7 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx); void hisi_qm_free_qp(handle_t h_qp);

/** - * hisi_check_bd_id - Check the SQE BD's id and send msg id. + * hisi_check_bd_id - Check the SQE BD's id and send msg id * @h_qp: Handle of the qp. * @mid: send message id. * @bid: recv BD id. @@ -134,7 +135,7 @@ void hisi_qm_free_qp(handle_t h_qp); int hisi_check_bd_id(handle_t h_qp, __u32 mid, __u32 bid);

/** - * hisi_set_msg_id - set the message tag id. + * hisi_set_msg_id - set the alg message tag id * @h_qp: Handle of the qp. * @tag: the message tag id. */ diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 0a1bcc1..f008c16 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1,15 +1,15 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */

-#include <stdbool.h> -#include <stdlib.h> -#include <pthread.h> #include "drv/wd_cipher_drv.h" #include "drv/wd_digest_drv.h" #include "drv/wd_aead_drv.h" #include "crypto/aes.h" #include "crypto/galois.h" #include "hisi_qm_udrv.h" +#include "wd_cipher.h" +#include "wd_digest.h" +#include "wd_aead.h"

#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 @@ -523,76 +523,76 @@ static __u32 g_sec_hmac_full_len[WD_DIGEST_TYPE_MAX] = { SEC_HMAC_SHA512_MAC_LEN, SEC_HMAC_SHA512_224_MAC_LEN, SEC_HMAC_SHA512_256_MAC_LEN };

-static int hisi_sec_init(struct wd_alg_driver *drv, void *conf); -static void hisi_sec_exit(struct wd_alg_driver *drv); +static int hisi_sec_init(void *conf, void *priv); +static void hisi_sec_exit(void *priv);

-static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg);

-static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg);

-static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg);

-static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_send(drv, ctx, msg); - return hisi_sec_cipher_send_v3(drv, ctx, msg); + return hisi_sec_cipher_send(ctx, msg); + return hisi_sec_cipher_send_v3(ctx, msg); }

-static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_recv(drv, ctx, msg); - return hisi_sec_cipher_recv_v3(drv, ctx, msg); + return hisi_sec_cipher_recv(ctx, msg); + return hisi_sec_cipher_recv_v3(ctx, msg); }

-static int digest_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_send(drv, ctx, msg); - return hisi_sec_digest_send_v3(drv, ctx, msg); + return hisi_sec_digest_send(ctx, msg); + return hisi_sec_digest_send_v3(ctx, msg); }

-static int digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_recv(drv, ctx, msg); - return hisi_sec_digest_recv_v3(drv, ctx, msg); + return hisi_sec_digest_recv(ctx, msg); + return hisi_sec_digest_recv_v3(ctx, msg); }

-static int aead_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_send(drv, ctx, msg); - return hisi_sec_aead_send_v3(drv, ctx, msg); + return hisi_sec_aead_send(ctx, msg); + return hisi_sec_aead_send_v3(ctx, msg); }

-static int aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx);

if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_recv(drv, ctx, msg); - return hisi_sec_aead_recv_v3(drv, ctx, msg); + return hisi_sec_aead_recv(ctx, msg); + return hisi_sec_aead_recv_v3(ctx, msg); }

static int hisi_sec_get_usage(void *param) @@ -606,6 +606,7 @@ static int hisi_sec_get_usage(void *param) .alg_name = (sec_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_sec_ctx),\ .queue_num = SEC_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ .fallback = 0,\ @@ -1141,7 +1142,7 @@ static int fill_cipher_bd2(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) return 0; }

-static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1167,6 +1168,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w }

hisi_set_msg_id(h_qp, &msg->tag); + sqe.type2.clen_ivhlen |= (__u32)msg->in_bytes; sqe.type2.tag = (__u16)msg->tag; fill_cipher_bd2_addr(msg, &sqe); @@ -1186,7 +1188,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w return 0; }

-static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1339,12 +1341,13 @@ static int fill_cipher_bd3(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) return ret; }

- sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; + sqe->auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET);

return 0; }

-static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1370,6 +1373,7 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void }

hisi_set_msg_id(h_qp, &msg->tag); + sqe.c_len_ivin = (__u32)msg->in_bytes; sqe.tag = (__u64)(uintptr_t)msg->tag; fill_cipher_bd3_addr(msg, &sqe); @@ -1377,7 +1381,7 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("cipher send sqe is err(%d)!\n", ret); + WD_ERR("cipher send sqe v3 is err(%d)!\n", ret);

if (msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, @@ -1434,7 +1438,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "cipher"); }

-static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1688,7 +1692,7 @@ static int digest_len_check(struct wd_digest_msg *msg, enum sec_bd_type type) return 0; }

-static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -1755,7 +1759,7 @@ put_sgl: return ret; }

-static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -1932,7 +1936,7 @@ static void fill_digest_v3_scene(struct hisi_sec_sqe3 *sqe, sqe->bd_param |= (__u16)(de | scene); }

-static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -1975,12 +1979,13 @@ static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void

hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = (__u64)(uintptr_t)msg->tag; - sqe.auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; + sqe.auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET);

ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("digest send sqe is err(%d)!\n", ret); + WD_ERR("digest send sqe v3 is err(%d)!\n", ret);

goto put_sgl; } @@ -2031,7 +2036,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "digest"); }

-static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -2524,7 +2529,7 @@ int aead_msg_state_check(struct wd_aead_msg *msg) return 0; }

-static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *msg = wd_msg; @@ -2647,13 +2652,22 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) { /* Asynchronous mode does not use the sent message, so ignores it */ - if (msg->cmode == WD_CIPHER_GCM) - return (msg->msg_state == AEAD_MSG_END) && qp->q_info.qp_mode == CTX_MODE_SYNC; + if (qp->q_info.qp_mode == CTX_MODE_ASYNC) + return false; + /* + * For aead gcm stream mode, due to some hardware limitations, + * the final message was not sent to hardware if the qm is + * not higher than v3 version or the input length of the + * message is 0, the software calculation has been executed. + */ + if (msg->msg_state == AEAD_MSG_END && msg->cmode == WD_CIPHER_GCM && + (qp->q_info.hw_type <= HISI_QM_API_VER3_BASE || !msg->in_bytes)) + return true;

return false; }

-static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -2903,7 +2917,6 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) sqe->c_len_ivin = msg->in_bytes; sqe->cipher_src_offset = msg->assoc_bytes; sqe->a_len_key = msg->in_bytes + msg->assoc_bytes; - sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET;

ret = fill_aead_bd3_alg(msg, sqe); if (ret) { @@ -2917,10 +2930,13 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) return ret; }

+ sqe->auth_mac_key |= (__u32)(SEC_ENABLE_SVA_PREFETCH << + SEC_SVA_PREFETCH_OFFSET); + return 0; }

-static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *msg = wd_msg; @@ -2954,6 +2970,7 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * }

fill_aead_bd3_addr(msg, &sqe); + ret = fill_stream_bd3(h_qp, msg, &sqe); if (ret == WD_SOFT_COMPUTING) { ret = 0; @@ -2967,7 +2984,7 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) - WD_ERR("aead send sqe is err(%d)!\n", ret); + WD_ERR("aead send sqe v3 is err(%d)!\n", ret);

goto put_sgl; } @@ -3024,7 +3041,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "aead"); }

-static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3052,11 +3069,11 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * return 0; }

-static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) +static int hisi_sec_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_sec_ctx *sec_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_sec_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -3066,10 +3083,6 @@ static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; }

- priv = malloc(sizeof(struct hisi_sec_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { @@ -3086,8 +3099,7 @@ static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) goto out; config->ctxs[i].sqn = qm_priv.sqn; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sec_ctx->config, config, sizeof(struct wd_ctx_config_internal));

return 0;

@@ -3096,29 +3108,26 @@ out: h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); hisi_qm_free_qp(h_qp); } - free(priv); return -WD_EINVAL; }

-static void hisi_sec_exit(struct wd_alg_driver *drv) +static void hisi_sec_exit(void *priv) { - struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; + struct hisi_sec_ctx *sec_ctx = priv; struct wd_ctx_config_internal *config; handle_t h_qp; __u32 i;

if (!priv) { - /* return if already exit */ + WD_ERR("invalid: input parameter is NULL!\n"); return; }

- config = &priv->config; + config = &sec_ctx->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; }

#ifdef WD_STATIC_DRV @@ -3179,3 +3188,4 @@ static void __attribute__((destructor)) hisi_sec2_remove(void) for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&aead_alg_driver[i]); } + diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 99cd640..c8812df 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -22,10 +22,10 @@ typedef void (sm3_ce_block_fn)(__u32 word_reg[SM3_STATE_WORDS], const unsigned char *src, size_t blocks);

-static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf); -static void sm3_ce_drv_exit(struct wd_alg_driver *drv); -static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); -static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); +static int sm3_ce_drv_init(void *conf, void *priv); +static void sm3_ce_drv_exit(void *priv); +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg); +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg); static int sm3_ce_get_usage(void *param);

static struct wd_alg_driver sm3_ce_alg_driver = { @@ -33,6 +33,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, .priority = 200, + .priv_size = sizeof(struct sm3_ce_drv_ctx), .queue_num = 1, .op_type_num = 1, .fallback = 0, @@ -337,7 +338,7 @@ static int do_hmac_sm3_ce(struct wd_digest_msg *msg, __u8 *out_hmac) return WD_SUCCESS; }

-static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) { struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; __u8 digest[SM3_DIGEST_SIZE] = {0}; @@ -365,38 +366,26 @@ static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest return ret; }

-static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg) { return WD_SUCCESS; }

-static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf) +static int sm3_ce_drv_init(void *conf, void *priv) { - struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; - struct sm3_ce_drv_ctx *priv; + struct wd_ctx_config_internal *config = conf; + struct sm3_ce_drv_ctx *sctx = priv;

/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;

- priv = malloc(sizeof(struct sm3_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal));

return WD_SUCCESS; }

-static void sm3_ce_drv_exit(struct wd_alg_driver *drv) +static void sm3_ce_drv_exit(void *priv) { - struct sm3_ce_drv_ctx *sctx = (struct sm3_ce_drv_ctx *)drv->priv; - - if (!sctx) - return; - - free(sctx); - drv->priv = NULL; } diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 3404465..6475246 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -8,12 +8,12 @@ * https://www.openssl.org/source/license.html */ /* - * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */

#include "drv/wd_cipher_drv.h" -#include "wd_cipher.h" #include "isa_ce_sm4.h" +#include "wd_cipher.h"

#define SM4_ENCRYPT 1 #define SM4_DECRYPT 0 @@ -31,35 +31,23 @@ ((p)[0] = (__u8)((v) >> 24), (p)[1] = (__u8)((v) >> 16), \ (p)[2] = (__u8)((v) >> 8), (p)[3] = (__u8)(v))

-static int isa_ce_init(struct wd_alg_driver *drv, void *conf) +static int isa_ce_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct sm4_ce_drv_ctx *priv; + struct sm4_ce_drv_ctx *sctx = priv;

/* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0;

- priv = malloc(sizeof(struct sm4_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal));

- return WD_SUCCESS; + return 0; }

-static void isa_ce_exit(struct wd_alg_driver *drv) +static void isa_ce_exit(void *priv) { - struct sm4_ce_drv_ctx *sctx = (struct sm4_ce_drv_ctx *)drv->priv; - - if (!sctx) - return; - - free(sctx); - drv->priv = NULL; }

/* increment upper 96 bits of 128-bit counter by 1 */ @@ -333,7 +321,7 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey return 0; }

-static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; @@ -399,19 +387,19 @@ static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ return ret; }

-static int isa_ce_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { return 0; }

-static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { - return isa_ce_cipher_send(drv, ctx, msg); + return isa_ce_cipher_send(ctx, msg); }

-static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { - return isa_ce_cipher_recv(drv, ctx, msg); + return isa_ce_cipher_recv(ctx, msg); }

#define GEN_CE_ALG_DRIVER(ce_alg_name, alg_type) \ @@ -420,6 +408,7 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ .priority = 200,\ + .priv_size = sizeof(struct sm4_ce_drv_ctx),\ .op_type_num = 1,\ .fallback = 0,\ .init = isa_ce_init,\ diff --git a/include/wd.h b/include/wd.h index 21e5e06..77aac72 100644 --- a/include/wd.h +++ b/include/wd.h @@ -35,13 +35,6 @@ extern "C" { typedef unsigned char __u8; typedef unsigned int __u32; typedef unsigned long long __u64; -/* Required compiler attributes */ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -#define handle_t uintptr_t -typedef struct wd_dev_mask wd_dev_mask_t; - typedef void (*wd_log)(const char *format, ...);

#ifndef WD_NO_LOG @@ -104,12 +97,7 @@ typedef void (*wd_log)(const char *format, ...); #define WD_IS_ERR(h) ((uintptr_t)(h) > \ (uintptr_t)(-1000))

-enum wd_buff_type { - WD_FLAT_BUF, - WD_SGL_BUF, -}; - -enum wd_alg_type { +enum wcrypto_type { WD_CIPHER, WD_DIGEST, WD_AEAD, @@ -155,6 +143,9 @@ struct wd_dev_mask { unsigned int magic; };

+#define handle_t uintptr_t +typedef struct wd_dev_mask wd_dev_mask_t; + #if defined(__AARCH64_CMODEL_SMALL__) && __AARCH64_CMODEL_SMALL__ #define dsb(opt) { asm volatile("dsb " #opt : : : "memory"); } #define rmb() dsb(ld) /* read fence */ @@ -194,14 +185,14 @@ static inline void wd_iowrite64(void *addr, uint64_t value) *((volatile uint64_t *)addr) = value; }

-static inline void *WD_ERR_PTR(uintptr_t error) +static inline void *WD_ERR_PTR(intptr_t error) { return (void *)error; }

static inline long WD_PTR_ERR(const void *ptr) { - return (long)ptr; + return (intptr_t)ptr; }

/** diff --git a/include/wd_aead.h b/include/wd_aead.h index 01f6980..c17cd1d 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -8,6 +8,7 @@ #define __WD_AEAD_H

#include <dlfcn.h> +#include "wd.h" #include "wd_alg_common.h" #include "wd_cipher.h" #include "wd_digest.h" diff --git a/include/wd_alg.h b/include/wd_alg.h index aba855d..6203893 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -62,7 +62,7 @@ extern "C" { # define HWCAP2_RNG (1 << 16) #endif

-enum alg_dev_type { +enum alg_priority { UADK_ALG_SOFT = 0x0, UADK_ALG_CE_INSTR = 0x1, UADK_ALG_SVE_INSTR = 0x2, @@ -104,38 +104,18 @@ struct wd_alg_driver { int calc_type; int queue_num; int op_type_num; - void *priv; + int priv_size; handle_t fallback;

- int (*init)(struct wd_alg_driver *drv, void *conf); - void (*exit)(struct wd_alg_driver *drv); - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*init)(void *conf, void *priv); + void (*exit)(void *priv); + int (*send)(handle_t ctx, void *drv_msg); + int (*recv)(handle_t ctx, void *drv_msg); int (*get_usage)(void *param); int (*get_extend_ops)(void *ops); };

-inline int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) -{ - return drv->init(drv, conf); -} - -inline void wd_alg_driver_exit(struct wd_alg_driver *drv) -{ - drv->exit(drv); -} - -inline int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->send(drv, ctx, msg); -} - -inline int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->recv(drv, ctx, msg); -} - -/* +/** * wd_alg_driver_register() - Register a device driver. * @wd_alg_driver: a device driver that supports an algorithm. * diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index fd77426..d1c0038 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -10,6 +10,8 @@ #include <pthread.h> #include <stdbool.h> #include <numa.h> +#include <asm/types.h> + #include "wd.h" #include "wd_alg.h"

@@ -17,10 +19,11 @@ extern "C" { #endif

-#define BYTE_BITS 8 -#define BYTE_BITS_SHIFT 3 -#define GET_NEGATIVE(val) (0 - (val)) +/* Required compiler attributes */ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0)

+#define BYTE_BITS_SHIFT 3 #define BITS_TO_BYTES(bits) (((bits) + 7) >> 3) #define BYTES_TO_BITS(bytes) ((bytes) << 3)

@@ -55,12 +58,17 @@ enum wd_ctx_mode { CTX_MODE_MAX, };

+enum wd_buff_type { + WD_FLAT_BUF, + WD_SGL_BUF, +}; + enum wd_init_type { WD_TYPE_V1, WD_TYPE_V2, };

-/* +/** * struct wd_ctx - Define one ctx and related type. * @ctx: The ctx itself. * @op_type: Define the operation type of this specific ctx. @@ -74,7 +82,7 @@ struct wd_ctx { __u8 ctx_mode; };

-/* +/** * struct wd_cap_config - Capabilities. * @ctx_msg_num: number of asynchronous msg pools that the user wants to allocate. * Optional, user can set ctx_msg_num based on the number of requests @@ -87,7 +95,7 @@ struct wd_cap_config { __u32 resv; };

-/* +/** * struct wd_ctx_config - Define a ctx set and its related attributes, which * will be used in the scope of current process. * @ctx_num: The ctx number in below ctx array. @@ -103,7 +111,7 @@ struct wd_ctx_config { struct wd_cap_config *cap; };

-/* +/** * struct wd_ctx_nums - Define the ctx sets numbers. * @sync_ctx_num: The ctx numbers which are used for sync mode for each * ctx sets. @@ -115,7 +123,7 @@ struct wd_ctx_nums { __u32 async_ctx_num; };

-/* +/** * struct wd_ctx_params - Define the ctx sets params which are used for init * algorithms. * @op_type_num: Used for index of ctx_set_num, the order is the same as @@ -153,11 +161,11 @@ struct wd_ctx_config_internal { unsigned long *msg_cnt; };

-/* +/** * struct wd_comp_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling - * @sched_init: inited the scheduler input parameters. + * @sched_init: inited the scheduler input parameters. * @pick_next_ctx: Pick the proper ctx which a request will be sent to. * config points to the ctx config; sched_ctx points to * scheduler context; req points to the request. Return diff --git a/include/wd_comp.h b/include/wd_comp.h index 45994ff..c5dcefa 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -7,8 +7,7 @@ #ifndef __WD_COMP_H #define __WD_COMP_H

-#include <numa.h> - +#include "wd.h" #include "wd_alg_common.h"

#ifdef __cplusplus diff --git a/include/wd_digest.h b/include/wd_digest.h index f0916c3..d88a66c 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -8,6 +8,7 @@ #define __WD_DIGEST_H #include <dlfcn.h>

+#include "wd.h" #include "wd_alg_common.h"

#ifdef __cplusplus @@ -144,12 +145,12 @@ struct wd_digest_req { };

struct wd_cb_tag { - void *ctx; /* user: context or other user relatives */ - void *tag; /* to store user tag */ - int ctx_id; /* user id: context ID or other user identifier */ + void *ctx; /* user: context or other user relatives */ + void *tag; /* to store user tag */ + int ctx_id; /* user id: context ID or other user identifier */ };

-/* Digest tag format */ +/* Digest tag format of warpdrive */ struct wd_digest_tag { struct wd_cb_tag wd_tag; __u64 long_data_len; diff --git a/include/wd_util.h b/include/wd_util.h index dc1e41f..63aa5af 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -16,16 +16,17 @@ #include "wd.h" #include "wd_sched.h" #include "wd_alg.h" +#include "wd_alg_common.h"

#ifdef __cplusplus extern "C" { #endif

-#define WD_POOL_MAX_ENTRIES 1024 +#define WD_POOL_MAX_ENTRIES 1024

#define FOREACH_NUMA(i, config, config_numa) \ for ((i) = 0, (config_numa) = (config)->config_per_numa; \ - (i) < (config)->numa_num; (config_numa)++, (i)++) + (i) < (config)->numa_num; (config_numa)++, (i)++)

enum wd_status { WD_UNINIT, @@ -118,8 +119,8 @@ struct wd_ctx_attr { };

struct wd_msg_handle { - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*send)(handle_t sess, void *msg); + int (*recv)(handle_t sess, void *msg); };

struct wd_init_attrs { @@ -376,7 +377,6 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en);

/** * wd_handle_msg_sync() - recv msg from hardware - * @drv: the driver to handle msg. * @msg_handle: callback of msg handle ops. * @ctx: the handle of context. * @msg: the msg of task. @@ -385,8 +385,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); * * Return 0 if successful or less than 0 otherwise. */ -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en); +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en);

/** * wd_init_check() - Check input parameters for wd_<alg>_init. @@ -443,11 +443,11 @@ static inline void wd_alg_clear_init(enum wd_status *status) /** * wd_ctx_param_init() - Initialize the current device driver according * to the obtained queue resource and the applied driver. - * @ctx_params: wd_ctx_params to be initialized. - * @user_ctx_params: user input wd_ctx_params. - * @driver: device driver for the current algorithm application. + * @ctx_params: queue configuration parameters required for device initialization. + * @user_ctx_params: user-configured queue context parameters. + * @driver: device drivers that support current business algorithms. * @type: algorithm type. - * @max_op_type: algorithm max operation type. + * @max_op_type: number of device queue types that support algorithmic services. * * Return 0 if succeed and other error number if fail. */ @@ -484,13 +484,14 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv); * to the obtained queue resource and the applied driver. * @config: device resources requested by the current algorithm. * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. * * Return 0 if succeed and other error number if fail. */ int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); + struct wd_alg_driver *driver, void **drv_priv); void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); + struct wd_alg_driver *driver, void **drv_priv);

/** * wd_dlopen_drv() - Open the dynamic library file of the device driver. diff --git a/wd.c b/wd.c index 6d01158..2da5a5f 100644 --- a/wd.c +++ b/wd.c @@ -18,8 +18,10 @@ #include <numa.h> #include <sched.h>

-#include "wd.h" +#include "wd_alg_common.h" #include "wd_alg.h" +#include "wd.h" + #define SYS_CLASS_DIR "/sys/class/uacce" #define FILE_MAX_SIZE (8 << 20)

@@ -202,7 +204,7 @@ int wd_is_isolate(struct uacce_dev *dev) int value = 0; int ret;

- if (!dev || !strlen(dev->dev_root)) + if (!dev) return -WD_EINVAL;

ret = access_attr(dev->dev_root, "isolate", F_OK); @@ -282,12 +284,12 @@ static struct uacce_dev *read_uacce_sysfs(const char *dev_name) return NULL;

ret = snprintf(dev->dev_root, MAX_DEV_NAME_LEN, "%s/%s", - SYS_CLASS_DIR, dev_name); + SYS_CLASS_DIR, dev_name); if (ret < 0) goto out;

ret = snprintf(dev->char_dev_path, MAX_DEV_NAME_LEN, - "/dev/%s", dev_name); + "/dev/%s", dev_name); if (ret < 0) goto out;

@@ -703,7 +705,7 @@ struct uacce_dev_list *wd_get_accel_list(const char *alg_name) ret = access_attr(SYS_CLASS_DIR, dev_dir->d_name, F_OK); if (ret < 0) { WD_ERR("failed to access dev: %s, ret: %d\n", - dev_dir->d_name, ret); + dev_dir->d_name, ret); continue; }

diff --git a/wd_alg.c b/wd_alg.c index 3d014e1..5c5c665 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -5,9 +5,9 @@

#define _GNU_SOURCE #include <dirent.h> -#include <errno.h> #include <stdbool.h> #include <stdlib.h> +#include <string.h> #include <pthread.h> #include <sys/auxv.h>

diff --git a/wd_cipher.c b/wd_cipher.c index f6b035a..a833074 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -53,6 +53,7 @@ struct wd_cipher_setting { struct wd_sched sched; struct wd_async_msg_pool pool; struct wd_alg_driver *driver; + void *priv; void *dlhandle; void *dlh_list; } wd_cipher_setting; @@ -346,7 +347,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, goto out_clear_sched;

ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + wd_cipher_setting.driver, + &wd_cipher_setting.priv); if (ret) goto out_clear_pool;

@@ -363,10 +365,9 @@ out_clear_ctx_config:

static int wd_cipher_common_uninit(void) { - enum wd_status status; + void *priv = wd_cipher_setting.priv;

- wd_alg_get_init(&wd_cipher_setting.status, &status); - if (status == WD_UNINIT) + if (!priv) return -WD_EINVAL;

/* uninit async request pool */ @@ -376,7 +377,8 @@ static int wd_cipher_common_uninit(void) wd_clear_sched(&wd_cipher_setting.sched);

wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + wd_cipher_setting.driver, + &wd_cipher_setting.priv);

return 0; } @@ -588,9 +590,10 @@ static int cipher_iv_len_check(struct wd_cipher_req *req, return ret; }

-static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) +static int cipher_in_len_check(handle_t h_sess, struct wd_cipher_req *req) { struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + int ret = 0;

if (!req->in_bytes) { WD_ERR("invalid: cipher input length is zero!\n"); @@ -600,14 +603,27 @@ static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) if (sess->alg != WD_CIPHER_AES && sess->alg != WD_CIPHER_SM4) return 0;

- if ((req->in_bytes & (AES_BLOCK_SIZE - 1)) && - (sess->mode == WD_CIPHER_CBC || sess->mode == WD_CIPHER_ECB)) { - WD_ERR("failed to check input bytes of AES or SM4, size = %u\n", - req->in_bytes); - return -WD_EINVAL; + switch (sess->mode) { + case WD_CIPHER_ECB: + case WD_CIPHER_CBC: + if (req->in_bytes & (AES_BLOCK_SIZE - 1)) + ret = -WD_EINVAL; + break; + case WD_CIPHER_CBC_CS1: + case WD_CIPHER_CBC_CS2: + case WD_CIPHER_CBC_CS3: + if (req->in_bytes < AES_BLOCK_SIZE) + ret = -WD_EINVAL; + break; + default: + break; }

- return 0; + if (ret) + WD_ERR("invalid: %s input bytes is %u!\n", + wd_cipher_alg_name[sess->alg][sess->mode], req->in_bytes); + + return ret; }

static int wd_cipher_check_params(handle_t h_sess, @@ -632,7 +648,7 @@ static int wd_cipher_check_params(handle_t h_sess, return -WD_EINVAL; }

- ret = cipher_len_check(h_sess, req); + ret = cipher_in_len_check(h_sess, req); if (unlikely(ret)) return ret;

@@ -672,8 +688,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, msg_handle.recv = wd_cipher_setting.driver->recv;

wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, + wd_cipher_setting.config.epoll_en); wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type);

return ret; @@ -748,7 +764,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id;

- ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = wd_cipher_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -797,7 +813,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = wd_cipher_setting.driver->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { diff --git a/wd_comp.c b/wd_comp.c index 0fa5f92..9d25ce5 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -12,7 +12,10 @@ #include <string.h> #include <time.h>

+#include "config.h" #include "drv/wd_comp_drv.h" +#include "wd_sched.h" +#include "wd_util.h" #include "wd_comp.h"

#define HW_CTX_SIZE (64 * 1024) @@ -47,6 +50,7 @@ struct wd_comp_setting { struct wd_sched sched; struct wd_async_msg_pool pool; struct wd_alg_driver *driver; + void *priv; void *dlhandle; void *dlh_list; } wd_comp_setting; @@ -146,8 +150,7 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc { int ret;

- ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", - &wd_comp_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", &wd_comp_setting.config.epoll_en); if (ret < 0) return ret;

@@ -166,7 +169,8 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc goto out_clear_sched;

ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_comp_setting.driver, + &wd_comp_setting.priv); if (ret) goto out_clear_pool;

@@ -183,10 +187,9 @@ out_clear_ctx_config:

static int wd_comp_uninit_nolock(void) { - enum wd_status status; + void *priv = wd_comp_setting.priv;

- wd_alg_get_init(&wd_comp_setting.status, &status); - if (status == WD_UNINIT) + if (!priv) return -WD_EINVAL;

/* Uninit async request pool */ @@ -196,7 +199,8 @@ static int wd_comp_uninit_nolock(void) wd_clear_sched(&wd_comp_setting.sched);

wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_comp_setting.driver, + &wd_comp_setting.priv);

return 0; } @@ -377,7 +381,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_alg_driver_recv(wd_comp_setting.driver, ctx->ctx, &resp_msg); + ret = wd_comp_setting.driver->recv(ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -611,8 +615,8 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, msg_handle.recv = wd_comp_setting.driver->recv;

pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_comp_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock);

return ret; @@ -866,12 +870,18 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS;

- ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); + pthread_spin_lock(&ctx->lock); + + ret = wd_comp_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { - WD_ERR("wd comp send error, ret = %d!\n", ret); + pthread_spin_unlock(&ctx->lock); + if (ret != -WD_EBUSY) + WD_ERR("wd comp send error, ret = %d!\n", ret); goto fail_with_msg; }

+ pthread_spin_unlock(&ctx->lock); + wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_comp_env_config, idx); if (unlikely(ret)) diff --git a/wd_digest.c b/wd_digest.c index 58f621a..9fb9c9b 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -7,11 +7,12 @@ #include <stdlib.h> #include <pthread.h> #include <limits.h> +#include "wd_util.h" #include "include/drv/wd_digest_drv.h" #include "wd_digest.h"

#define GMAC_IV_LEN 16 -#define MAX_BLOCK_SIZE 128 +#define MAX_BLOCK_SIZE 128U

static __u32 g_digest_mac_len[WD_DIGEST_TYPE_MAX] = { WD_DIGEST_SM3_LEN, WD_DIGEST_MD5_LEN, WD_DIGEST_SHA1_LEN, @@ -42,6 +43,7 @@ struct wd_digest_setting { struct wd_sched sched; struct wd_alg_driver *driver; struct wd_async_msg_pool pool; + void *priv; void *dlhandle; void *dlh_list; } wd_digest_setting; @@ -277,7 +279,8 @@ static int wd_digest_init_nolock(struct wd_ctx_config *config, goto out_clear_sched;

ret = wd_alg_init_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + wd_digest_setting.driver, + &wd_digest_setting.priv); if (ret) goto out_clear_pool;

@@ -326,29 +329,21 @@ out_clear_init: return ret; }

-static int wd_digest_uninit_nolock(void) +static void wd_digest_uninit_nolock(void) { - enum wd_status status; - - wd_alg_get_init(&wd_digest_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_digest_setting.pool); wd_clear_sched(&wd_digest_setting.sched); wd_alg_uninit_driver(&wd_digest_setting.config, - wd_digest_setting.driver); - return 0; + wd_digest_setting.driver, + &wd_digest_setting.priv); }

void wd_digest_uninit(void) { - int ret; - - ret = wd_digest_uninit_nolock(); - if (ret) + if (!wd_digest_setting.priv) return;

+ wd_digest_uninit_nolock(); wd_digest_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_digest_setting.status); } @@ -391,7 +386,6 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, if (state) goto out_uninit;

- while (ret != 0) { memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal));

@@ -450,12 +444,10 @@ out_uninit:

void wd_digest_uninit2(void) { - int ret; - - ret = wd_digest_uninit_nolock(); - if (ret) + if (!wd_digest_setting.priv) return;

+ wd_digest_uninit_nolock(); wd_alg_attrs_uninit(&wd_digest_init_attrs); wd_alg_drv_unbind(wd_digest_setting.driver); wd_digest_close_driver(WD_TYPE_V2); @@ -618,8 +610,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds msg_handle.recv = wd_digest_setting.driver->recv;

wd_ctx_spin_lock(ctx, wd_digest_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_digest_setting.config.epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, wd_digest_setting.config.epoll_en); wd_ctx_spin_unlock(ctx, wd_digest_setting.driver->calc_type); if (unlikely(ret)) return ret; @@ -713,7 +705,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) fill_request_msg(msg, req, dsess); msg->tag = msg_id;

- ret = wd_alg_driver_send(wd_digest_setting.driver, ctx->ctx, msg); + ret = wd_digest_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -762,7 +754,8 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_alg_driver_recv(wd_digest_setting.driver, ctx->ctx, &recv_msg); + ret = wd_digest_setting.driver->recv(ctx->ctx, + &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { diff --git a/wd_sched.c b/wd_sched.c index aa6c91e..36bf71e 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -499,7 +499,8 @@ static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) } }

-int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) +int wd_sched_rr_instance(const struct wd_sched *sched, + struct sched_params *param) { struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; diff --git a/wd_util.c b/wd_util.c index d58e8b8..2dd81c9 100644 --- a/wd_util.c +++ b/wd_util.c @@ -24,11 +24,10 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b))

#define WD_INIT_SLEEP_UTIME 1000 -#define WD_INIT_RETRY_TIMES 10000 +#define WD_INIT_RETRY_TIMES 1000 #define US2S(us) ((us) >> 20) #define WD_INIT_RETRY_TIMEOUT 3 - -#define WD_SOFT_CTX_NUM 2 +#define WD_SOFT_CTX_NUM 2 #define WD_SOFT_SYNC_CTX 0 #define WD_SOFT_ASYNC_CTX 1

@@ -421,8 +420,15 @@ void wd_uninit_async_request_pool(struct wd_async_msg_pool *pool) void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, int ctx_idx, __u32 tag) { - struct msg_pool *p = &pool->pools[ctx_idx]; - __u32 msg_num = p->msg_num; + struct msg_pool *p; + __u32 msg_num; + + if ((__u32)ctx_idx > pool->pool_num) { + WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); + return NULL; + } + p = &pool->pools[ctx_idx]; + msg_num = p->msg_num;

/* tag value start from 1 */ if (tag == 0 || tag > msg_num) { @@ -1585,17 +1591,22 @@ static int wd_init_async_polling_thread_per_numa(struct wd_env_config *config, task_queue = queue_head; for (i = 0; i < num; task_queue++, i++) { ret = wd_init_one_task_queue(task_queue, alg_poll_ctx); - if (ret) { - for (j = 0; j < i; task_queue++, j++) - wd_uninit_one_task_queue(task_queue); - free(queue_head); - return ret; - } + if (ret) + goto uninit; }

config_numa->async_task_queue_array = (void *)queue_head;

return 0; + +uninit: + task_queue = queue_head; + for (j = 0; j < i; task_queue++, j++) + wd_uninit_one_task_queue(task_queue); + + free(queue_head); + + return ret; }

static void wd_uninit_async_polling_thread_per_numa(struct wd_env_config *cfg, @@ -1814,8 +1825,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en) return 0; }

-int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en) +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en) { __u64 timeout = WD_RECV_MAX_CNT_NOSLEEP; __u64 rx_cnt = 0; @@ -1824,7 +1835,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand if (balance) timeout = WD_RECV_MAX_CNT_SLEEP;

- ret = msg_handle->send(drv, ctx, msg); + ret = msg_handle->send(ctx, msg); if (unlikely(ret < 0)) { WD_ERR("failed to send msg to hw, ret = %d!\n", ret); return ret; @@ -1837,7 +1848,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand WD_ERR("wd ctx wait timeout(%d)!\n", ret); }

- ret = msg_handle->recv(drv, ctx, msg); + ret = msg_handle->recv(ctx, msg); if (ret != -WD_EAGAIN) { if (unlikely(ret < 0)) { WD_ERR("failed to recv msg: error = %d!\n", ret); @@ -1865,12 +1876,12 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) { if (!config || !config->ctxs || !config->ctxs[0].ctx) { - WD_ERR("invalid: config or config->ctxs is NULL!\n"); + WD_ERR("invalid: wd_ctx_config is NULL!\n"); return -WD_EINVAL; }

if (!sched) { - WD_ERR("invalid: sched is NULL!\n"); + WD_ERR("invalid: wd_sched is NULL!\n"); return -WD_EINVAL; }

@@ -1882,11 +1893,44 @@ int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) return 0; }

+int wd_alg_try_init(enum wd_status *status) +{ + enum wd_status expected; + __u32 count = 0; + bool ret; + + /* + * Here is aimed to protect the security of the initialization interface + * in the multi-thread scenario. Only one thread can get the WD_INITING + * status to initialize algorithm. Other thread will wait for the result. + * And the algorithm initialization interfaces is a liner process. + * So the initing thread will return a result to notify other thread go on. + */ + do { + expected = WD_UNINIT; + ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (expected == WD_INIT) { + WD_ERR("The algorithm has been initialized!\n"); + return -WD_EEXIST; + } + usleep(WD_INIT_SLEEP_UTIME); + + if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { + WD_ERR("The algorithm initialize wait timeout!\n"); + return -WD_ETIMEDOUT; + } + } while (!ret); + + return 0; +} + static void wd_get_alg_type(const char *alg_name, char *alg_type) { + int alg_nums = ARRAY_SIZE(alg_options); int i;

- for (i = 0; i < ARRAY_SIZE(alg_options); i++) { + for (i = 0; i < alg_nums; i++) { if (strcmp(alg_name, alg_options[i].name) == 0) { (void)strcpy(alg_type, alg_options[i].algtype); break; @@ -1946,10 +1990,21 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) }

int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) + struct wd_alg_driver *driver, void **drv_priv) { + void *priv; int ret;

+ if (!driver->priv_size) { + WD_ERR("invalid: driver priv ctx size is zero!\n"); + return -WD_EINVAL; + } + + /* Init ctx related resources in specific driver */ + priv = calloc(1, driver->priv_size); + if (!priv) + return -WD_ENOMEM; + if (!driver->init) { driver->fallback = 0; WD_ERR("driver have no init interface.\n"); @@ -1957,7 +2012,7 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, goto err_alloc; }

- ret = driver->init(driver, config); + ret = driver->init(config, priv); if (ret < 0) { WD_ERR("driver init failed.\n"); goto err_alloc; @@ -1970,23 +2025,31 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("soft alg driver init failed.\n"); } } + *drv_priv = priv;

return 0;

err_alloc: + free(priv); return ret; }

void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) + struct wd_alg_driver *driver, void **drv_priv) { + void *priv = *drv_priv;

- driver->exit(driver); + driver->exit(priv); /* Ctx config just need clear once */ wd_clear_ctx_config(config);

if (driver->fallback) wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); + + if (priv) { + free(priv); + *drv_priv = NULL; + } }

void wd_dlclose_drv(void *dlh_list) @@ -2200,11 +2263,11 @@ int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir)

if (is_dir) { len = snprintf(lib_path, PATH_MAX, "%s/%s", file_path, WD_DRV_LIB_DIR); - if (len >= PATH_MAX) + if (len < 0) return -WD_EINVAL; } else { len = snprintf(lib_path, PATH_MAX, "%s/%s/%s", file_path, WD_DRV_LIB_DIR, lib_file); - if (len >= PATH_MAX) + if (len < 0) return -WD_EINVAL; }

@@ -2350,31 +2413,6 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv) wd_release_drv(drv); }

-int wd_alg_try_init(enum wd_status *status) -{ - enum wd_status expected; - __u32 count = 0; - bool ret; - - do { - expected = WD_UNINIT; - ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); - if (expected == WD_INIT) { - WD_ERR("The algorithm has been initialized!\n"); - return -WD_EEXIST; - } - usleep(WD_INIT_SLEEP_UTIME); - - if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { - WD_ERR("The algorithm initialize wait timeout!\n"); - return -WD_ETIMEDOUT; - } - } while (!ret); - - return 0; -} - static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) { __u32 count = 0; @@ -2715,8 +2753,8 @@ free_ctxs:

static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) { - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); + free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); + free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); free(ctx_config->ctxs); }

Synchronize the code of the test tool UADK Tools to ensure that the test tool code is normal before adding new functions

Signed-off-by: Longfang Liu liulongfang@huawei.com --- uadk_tool/benchmark/hpre_protocol_data.h | 2 +- uadk_tool/benchmark/hpre_uadk_benchmark.c | 40 +++------------------ uadk_tool/benchmark/hpre_wd_benchmark.c | 11 +++--- uadk_tool/benchmark/sec_soft_benchmark.c | 4 --- uadk_tool/benchmark/sec_uadk_benchmark.c | 43 ++++------------------- uadk_tool/benchmark/sec_wd_benchmark.c | 11 +++--- uadk_tool/benchmark/trng_wd_benchmark.c | 3 +- uadk_tool/benchmark/uadk_benchmark.c | 10 +++--- uadk_tool/benchmark/zip_uadk_benchmark.c | 37 ++----------------- uadk_tool/benchmark/zip_wd_benchmark.c | 3 +- 10 files changed, 36 insertions(+), 128 deletions(-)

diff --git a/uadk_tool/benchmark/hpre_protocol_data.h b/uadk_tool/benchmark/hpre_protocol_data.h index 7bdb942..fdbac35 100644 --- a/uadk_tool/benchmark/hpre_protocol_data.h +++ b/uadk_tool/benchmark/hpre_protocol_data.h @@ -246,7 +246,7 @@ static unsigned char rsa_qinv_3072[] = { };

static unsigned char rsa_d_3072[] = { - 0x33, 0x49, 0x58, 0xb0, 0x89, 0x3d, 0x96, 0x78, 0xb7, 0x2a, 0x99, 0x5e, 0x0c, 0x56, 0xae, 0x08, + 0x33, 0x49, 0x58, 0xb0, 0x89, 0x3d, 0x96, 0x78, 0xb7, 0x2a, 0x99, 0x5e, 0x0c, 0x56, 0xae, 0x08, 0x32, 0xe1, 0xb9, 0x1c, 0x17, 0xd7, 0x71, 0xaa, 0x57, 0x37, 0x58, 0xbf, 0x94, 0xf8, 0xa0, 0xdf, 0xdb, 0x0a, 0xfd, 0x2f, 0xce, 0x8a, 0x82, 0x43, 0x11, 0x48, 0x6b, 0xec, 0x61, 0x72, 0x77, 0x98, 0x53, 0x00, 0x32, 0x5e, 0xbe, 0x3d, 0x37, 0xaf, 0x53, 0x55, 0x13, 0xc7, 0x61, 0x03, 0x6a, 0x28, diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c index bc0687d..47ee5b5 100644 --- a/uadk_tool/benchmark/hpre_uadk_benchmark.c +++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c @@ -610,10 +610,7 @@ static void uninit_hpre_ctx_config2(int subtype)

static int init_hpre_ctx_config2(struct acc_option *options) { - struct wd_ctx_params cparams = {0}; - struct wd_ctx_nums *ctx_set_num; int subtype = options->subtype; - int mode = options->syncmode; char alg_name[MAX_ALG_NAME]; int ret;

@@ -623,49 +620,22 @@ static int init_hpre_ctx_config2(struct acc_option *options) return -EINVAL; }

- ctx_set_num = calloc(1, sizeof(*ctx_set_num)); - if (!ctx_set_num) { - WD_ERR("failed to alloc ctx_set_size!\n"); - return -WD_ENOMEM; - } - - cparams.op_type_num = 1; - cparams.ctx_set_num = ctx_set_num; - cparams.bmp = numa_allocate_nodemask(); - if (!cparams.bmp) { - WD_ERR("failed to create nodemask!\n"); - ret = -WD_ENOMEM; - goto out_freectx; - } - - numa_bitmask_setall(cparams.bmp); - - if (mode == CTX_MODE_SYNC) - ctx_set_num->sync_ctx_num = g_ctxnum; - else - ctx_set_num->async_ctx_num = g_ctxnum; - /* init2 */ switch (subtype) { case RSA_TYPE: - return wd_rsa_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + return wd_rsa_init2(alg_name, SCHED_POLICY_RR, TASK_HW); case DH_TYPE: - return wd_dh_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + return wd_dh_init2(alg_name, SCHED_POLICY_RR, TASK_HW); case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: - return wd_ecc_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + return wd_ecc_init2(alg_name, SCHED_POLICY_RR, TASK_HW); default: HPRE_TST_PRT("failed to parse alg subtype on uninit2!\n"); return -EINVAL; } - -out_freectx: - free(ctx_set_num); - - return ret; }

/*-------------------------------uadk benchmark main code-------------------------------------*/ @@ -2263,7 +2233,7 @@ static void *ecc_uadk_sync_run(void *arg) memset(&req, 0, sizeof(req));

memset(&setup, 0, sizeof(setup)); - if (subtype != X448_TYPE && subtype != X25519_TYPE) { + if (subtype != X448_TYPE || subtype != X25519_TYPE) { ret = get_ecc_curve(&setup, cid); if (ret) return NULL; @@ -2422,7 +2392,7 @@ static void *ecc_uadk_async_run(void *arg) memset(&req, 0, sizeof(req));

memset(&setup, 0, sizeof(setup)); - if (subtype != X448_TYPE && subtype != X25519_TYPE) { + if (subtype != X448_TYPE || subtype != X25519_TYPE) { ret = get_ecc_curve(&setup, cid); if (ret) return NULL; diff --git a/uadk_tool/benchmark/hpre_wd_benchmark.c b/uadk_tool/benchmark/hpre_wd_benchmark.c index 0196e62..6e9f208 100644 --- a/uadk_tool/benchmark/hpre_wd_benchmark.c +++ b/uadk_tool/benchmark/hpre_wd_benchmark.c @@ -459,7 +459,8 @@ static int init_hpre_wd_queue(struct acc_option *options) PATH_STR_SIZE, "%s", options->device); if (ret < 0) { WD_ERR("failed to copy dev file path!\n"); - return -WD_EINVAL; + ret = -WD_EINVAL; + goto queue_out; } }

@@ -556,8 +557,8 @@ void *hpre_wd_poll(void *data) while (last_time) { recv = wd_poll_ctx(queue, expt); /* - * async mode poll easy to 100% with small package. - * SEC_TST_PRT("poll %d recv: %u!\n", i, recv); + * warpdrive async mode poll easy to 100% with small package. + * SEC_TST_PRT("warpdrive poll %d recv: %u!\n", i, recv); */

if (unlikely(recv < 0)) { @@ -2115,7 +2116,7 @@ static void *ecc_wd_sync_run(void *arg) queue = g_thread_queue.bd_res[pdata->td_id].queue;

memset(&setup, 0, sizeof(setup)); - if (subtype != X448_TYPE && subtype != X25519_TYPE) { + if (subtype != X448_TYPE || subtype != X25519_TYPE) { ret = get_ecc_curve(&setup, cid); if (ret) return NULL; @@ -2273,7 +2274,7 @@ static void *ecc_wd_async_run(void *arg) queue = g_thread_queue.bd_res[pdata->td_id].queue;

memset(&setup, 0, sizeof(setup)); - if (subtype != X448_TYPE && subtype != X25519_TYPE) { + if (subtype != X448_TYPE || subtype != X25519_TYPE) { ret = get_ecc_curve(&setup, cid); if (ret) return NULL; diff --git a/uadk_tool/benchmark/sec_soft_benchmark.c b/uadk_tool/benchmark/sec_soft_benchmark.c index 8fa523c..850a0db 100644 --- a/uadk_tool/benchmark/sec_soft_benchmark.c +++ b/uadk_tool/benchmark/sec_soft_benchmark.c @@ -280,7 +280,6 @@ static int sec_soft_param_parse(soft_thread *tddata, struct acc_option *options) mode = WD_CIPHER_CBC; tddata->evp_cipher = EVP_des_ede3_cbc(); break; -#ifndef OPENSSL_NO_SM4 case SM4_128_ECB: keysize = 16; mode = WD_CIPHER_ECB; @@ -310,7 +309,6 @@ static int sec_soft_param_parse(soft_thread *tddata, struct acc_option *options) keysize = 16; mode = WD_CIPHER_XTS; break; -#endif case AES_128_CCM: keysize = 16; mode = WD_CIPHER_CCM; @@ -356,7 +354,6 @@ static int sec_soft_param_parse(soft_thread *tddata, struct acc_option *options) mode = WD_CIPHER_CBC; tddata->evp_cipher = EVP_aes_256_cbc(); break; -#ifndef OPENSSL_NO_SM4 case SM4_128_CCM: keysize = 16; mode = WD_CIPHER_CCM; @@ -365,7 +362,6 @@ static int sec_soft_param_parse(soft_thread *tddata, struct acc_option *options) keysize = 16; mode = WD_CIPHER_GCM; break; -#endif case SM3_ALG: // digest mode is optype keysize = 4; mode = optype; diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index 411d0c4..c4e4e27 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -150,10 +150,10 @@ static int sec_uadk_param_parse(thread_data *tddata, struct acc_option *options) u32 out_bytes = 32; u8 keysize = 0; u8 ivsize = 0; - u8 dmode = 0; - u8 dalg = 0; - u8 mode = 0; - u8 alg = 0; + u8 dmode; + u8 dalg; + u8 mode; + u8 alg;

switch(algtype) { case AES_128_ECB: @@ -795,10 +795,7 @@ static void uninit_ctx_config2(int subtype)

static int init_ctx_config2(struct acc_option *options) { - struct wd_ctx_params cparams = {0}; - struct wd_ctx_nums *ctx_set_num; int subtype = options->subtype; - int mode = options->syncmode; char alg_name[MAX_ALG_NAME]; int ret;

@@ -808,32 +805,10 @@ static int init_ctx_config2(struct acc_option *options) return -EINVAL; }

- ctx_set_num = calloc(1, sizeof(*ctx_set_num)); - if (!ctx_set_num) { - WD_ERR("failed to alloc ctx_set_size!\n"); - return -WD_ENOMEM; - } - - cparams.op_type_num = 1; - cparams.ctx_set_num = ctx_set_num; - cparams.bmp = numa_allocate_nodemask(); - if (!cparams.bmp) { - WD_ERR("failed to create nodemask!\n"); - ret = -WD_ENOMEM; - goto out_freectx; - } - - numa_bitmask_setall(cparams.bmp); - - if (mode == CTX_MODE_SYNC) - ctx_set_num->sync_ctx_num = g_ctxnum; - else - ctx_set_num->async_ctx_num = g_ctxnum; - /* init */ switch(subtype) { case CIPHER_TYPE: - ret = wd_cipher_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_cipher_init2(alg_name, SCHED_POLICY_RR, TASK_HW); if (ret) SEC_TST_PRT("failed to do cipher init2!\n"); break; @@ -843,12 +818,12 @@ static int init_ctx_config2(struct acc_option *options) SEC_TST_PRT("failed to do cipher intruction init2!\n"); break; case AEAD_TYPE: - ret = wd_aead_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_aead_init2(alg_name, SCHED_POLICY_RR, TASK_HW); if (ret) SEC_TST_PRT("failed to do aead init2!\n"); break; case DIGEST_TYPE: - ret = wd_digest_init2_(alg_name, options->sched_type, options->task_type, &cparams); + ret = wd_digest_init2(alg_name, options->sched_type, options->task_type); if (ret) SEC_TST_PRT("failed to do digest init2!\n"); break; @@ -858,11 +833,7 @@ static int init_ctx_config2(struct acc_option *options) return ret; }

-out_freectx: - free(ctx_set_num); - return ret; - }

static void get_aead_data(u8 *addr, u32 size) diff --git a/uadk_tool/benchmark/sec_wd_benchmark.c b/uadk_tool/benchmark/sec_wd_benchmark.c index e022dcb..86aef7a 100644 --- a/uadk_tool/benchmark/sec_wd_benchmark.c +++ b/uadk_tool/benchmark/sec_wd_benchmark.c @@ -214,10 +214,10 @@ static int sec_wd_param_parse(thread_data *tddata, struct acc_option *options) bool is_union = false; u8 keysize = 0; u8 ivsize = 0; - u8 dmode = 0; - u8 dalg = 0; - u8 mode = 0; - u8 alg = 0; + u8 dmode; + u8 dalg; + u8 mode; + u8 alg;

switch(algtype) { case AES_128_ECB: @@ -623,7 +623,8 @@ static int init_wd_queue(struct acc_option *options) PATH_STR_SIZE, "%s", options->device); if (ret < 0) { WD_ERR("failed to copy dev file path!\n"); - return -WD_EINVAL; + ret = -WD_EINVAL; + goto queue_out; } }

diff --git a/uadk_tool/benchmark/trng_wd_benchmark.c b/uadk_tool/benchmark/trng_wd_benchmark.c index 2f058d4..2fa437c 100644 --- a/uadk_tool/benchmark/trng_wd_benchmark.c +++ b/uadk_tool/benchmark/trng_wd_benchmark.c @@ -56,7 +56,8 @@ static int init_trng_wd_queue(struct acc_option *options) PATH_STR_SIZE, "%s", options->device); if (ret < 0) { WD_ERR("failed to copy dev file path!\n"); - return -WD_EINVAL; + ret = -WD_EINVAL; + goto free_mem; } }

diff --git a/uadk_tool/benchmark/uadk_benchmark.c b/uadk_tool/benchmark/uadk_benchmark.c index 0f01fdf..fcb28c4 100644 --- a/uadk_tool/benchmark/uadk_benchmark.c +++ b/uadk_tool/benchmark/uadk_benchmark.c @@ -480,7 +480,7 @@ void cal_perfermance_data(struct acc_option *option, u32 sttime) ops = perfops / option->times;

ACC_TST_PRT("algname:\tlength:\t\tperf:\t\tiops:\t\tCPU_rate:\n" - "%s\t%-2uBytes \t%.2f%s\t%.1fKops \t%.2f%%\n", + "%s\t%-2uBytes \t%.2f%s\t%.2fKops\t%.2f%%\n", palgname, option->pktlen, perfermance, unit, ops, cpu_rate); }

@@ -494,15 +494,13 @@ static int benchmark_run(struct acc_option *option) (option->modetype == INSTR_MODE) || (option->modetype == MULTIBUF_MODE)) { ret = sec_uadk_benchmark(option); + usleep(20000); } else if (option->modetype == NOSVA_MODE) { ret = sec_wd_benchmark(option); - } - usleep(20000); -#ifdef HAVE_CRYPTO - if (option->modetype == SOFT_MODE) { + usleep(20000); + } else if (option->modetype == SOFT_MODE) { ret = sec_soft_benchmark(option); } -#endif break; case HPRE_TYPE: if (option->modetype == SVA_MODE) { diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c index 2133297..1e98d95 100644 --- a/uadk_tool/benchmark/zip_uadk_benchmark.c +++ b/uadk_tool/benchmark/zip_uadk_benchmark.c @@ -17,6 +17,7 @@ #define COMPRESSION_RATIO_FACTOR 0.7 #define CHUNK_SIZE (128 * 1024) #define MAX_UNRECV_PACKET_NUM 2 + struct uadk_bd { u8 *src; u8 *dst; @@ -299,11 +300,8 @@ static void uninit_ctx_config2(void)

static int init_ctx_config2(struct acc_option *options) { - struct wd_ctx_params cparams = {0}; - struct wd_ctx_nums *ctx_set_num; - int mode = options->syncmode; char alg_name[MAX_ALG_NAME]; - int ret; + int ret = 0;

ret = get_alg_name(options->algtype, alg_name); if (ret) { @@ -311,43 +309,14 @@ static int init_ctx_config2(struct acc_option *options) return -EINVAL; }

- ctx_set_num = calloc(WD_DIR_MAX, sizeof(*ctx_set_num)); - if (!ctx_set_num) { - WD_ERR("failed to alloc ctx_set_size!\n"); - return -WD_ENOMEM; - } - - cparams.op_type_num = WD_DIR_MAX; - cparams.ctx_set_num = ctx_set_num; - cparams.bmp = numa_allocate_nodemask(); - if (!cparams.bmp) { - WD_ERR("failed to create nodemask!\n"); - ret = -WD_ENOMEM; - goto out_freectx; - } - - numa_bitmask_setall(cparams.bmp); - - for (int i = 0; i < WD_DIR_MAX; i++) { - if (mode == CTX_MODE_SYNC) - ctx_set_num[i].sync_ctx_num = g_ctxnum; - else - ctx_set_num[i].async_ctx_num = g_ctxnum; - } - /* init */ - ret = wd_comp_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_comp_init2(alg_name, SCHED_POLICY_RR, TASK_HW); if (ret) { ZIP_TST_PRT("failed to do comp init2!\n"); return ret; }

return 0; - -out_freectx: - free(ctx_set_num); - - return ret; }

static int specified_device_request_ctx(struct acc_option *options) diff --git a/uadk_tool/benchmark/zip_wd_benchmark.c b/uadk_tool/benchmark/zip_wd_benchmark.c index 8ad3e96..bdae79f 100644 --- a/uadk_tool/benchmark/zip_wd_benchmark.c +++ b/uadk_tool/benchmark/zip_wd_benchmark.c @@ -323,7 +323,8 @@ static int init_zip_wd_queue(struct acc_option *options) PATH_STR_SIZE, "%s", options->device); if (ret < 0) { WD_ERR("failed to copy dev file path!\n"); - return -WD_EINVAL; + ret = -WD_EINVAL; + goto queue_out; } }

Added a scheduler for heterogeneous computing. Added a dynamic scheduling solution. Balance the load of soft and hard computing to maintain the best performance

Signed-off-by: Longfang Liu liulongfang@huawei.com --- wd_sched.c | 661 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 631 insertions(+), 30 deletions(-)

diff --git a/wd_sched.c b/wd_sched.c index 36bf71e..b401b24 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -31,8 +31,11 @@ struct sched_key { int numa_id; __u8 type; __u8 mode; + __u8 ctx_prop; __u32 sync_ctxid; __u32 async_ctxid; + __u32 sw_sync_ctxid; + __u32 sw_async_ctxid; };

/* @@ -60,8 +63,31 @@ struct sched_ctx_region { * @valid: the region used flag. */ struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; + struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; // default as HW ctxs + struct sched_ctx_region *ce_ctx_region[SCHED_MODE_BUTT]; + struct sched_ctx_region *sve_ctx_region[SCHED_MODE_BUTT]; + struct sched_ctx_region *soft_ctx_region[SCHED_MODE_BUTT]; + bool nm_valid; + bool hw_valid; + bool ce_valid; + bool sve_valid; + bool soft_valid; +}; +#define SCHED_REGION_NUM 4 +#define LOOP_SWITH_TIME 5 + +enum sched_send_type { + SCHED_SEND_HW = 0, + SCHED_SEND_SW = 1 +}; + +struct wd_sched_balancer { + int switch_slice; + int next_send_type; + __u32 hw_task_num; + __u32 sw_task_num; + __u32 hw_dfx_num; + __u32 sw_dfx_num; };

/* @@ -79,6 +105,7 @@ struct wd_sched_ctx { __u16 numa_num; user_poll_func poll_func; int numa_map[NUMA_NUM_NODES]; + struct wd_sched_balancer balancer; struct wd_sched_info sched_info[0]; };

@@ -201,10 +228,11 @@ static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) skey->numa_id = param->numa_id; }

- if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + //if (skey->numa_id < 0) { + // WD_ERR("failed to get valid sched numa region!\n"); + // goto out; + //} + skey->numa_id = 0;

skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); @@ -331,7 +359,7 @@ static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 * while (++loop_time < MAX_POLL_TIMES) { for (i = 0; i < sched_ctx->numa_num;) { /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { + if (!sched_info[i].nm_valid) { i++; continue; } @@ -444,6 +472,446 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, return 0; }

+/* + * loop_get_ctx_range - Get ctx range from ctx_map by the wd comp arg + */ +static struct sched_ctx_region *loop_get_ctx_range( + struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +{ + struct wd_sched_info *sched_info; + int ctx_prop = key->ctx_prop; + int numa_id; + + sched_info = sched_ctx->sched_info; + if (key->numa_id >= 0 && sched_info[key->numa_id].hw_valid && ctx_prop == UADK_CTX_HW && + sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) + return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + else if (key->numa_id >= 0 && sched_info[key->numa_id].ce_valid && ctx_prop == UADK_CTX_CE_INS && + sched_info[key->numa_id].ce_ctx_region[key->mode][key->type].valid) + return &sched_info[key->numa_id].ce_ctx_region[key->mode][key->type]; + else if (key->numa_id >= 0 && sched_info[key->numa_id].sve_valid && ctx_prop == UADK_CTX_SVE_INS && + sched_info[key->numa_id].sve_ctx_region[key->mode][key->type].valid) + return &sched_info[key->numa_id].sve_ctx_region[key->mode][key->type]; + else if (key->numa_id >= 0 && sched_info[key->numa_id].soft_valid && ctx_prop == UADK_CTX_SOFT && + sched_info[key->numa_id].ce_ctx_region[key->mode][key->type].valid) + return &sched_info[key->numa_id].ce_ctx_region[key->mode][key->type]; + + /* If the key->numa_id is not exist, we should scan for a valid region */ + for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { + if (sched_info[numa_id].hw_valid) + return &sched_info[numa_id].ctx_region[key->mode][key->type]; + + if (sched_info[numa_id].ce_valid) + return &sched_info[numa_id].ce_ctx_region[key->mode][key->type]; + + if (sched_info[numa_id].sve_valid) + return &sched_info[numa_id].sve_ctx_region[key->mode][key->type]; + + if (sched_info[numa_id].soft_valid) + return &sched_info[numa_id].soft_ctx_region[key->mode][key->type]; + } + + return NULL; +} + +/* + * loop_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. + * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. + * @sched_key: The key of schedule region. + * @sched_mode: The sched async/sync mode. + * + * The user must init the schedule info through wd_sched_rr_instance + */ +static __u32 loop_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct sched_key *key, const int sched_mode) +{ + struct sched_ctx_region *region = NULL; + bool ret; + + key->mode = sched_mode; + ret = sched_key_valid(sched_ctx, key); + if (!ret) + return INVALID_POS; + + region = loop_get_ctx_range(sched_ctx, key); + if (!region) + return INVALID_POS; + + return sched_get_next_pos_rr(region, NULL); +} + +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + int cpu = sched_getcpu(); + int node = numa_node_of_cpu(cpu); + struct sched_key *skey; + int ctx_prop; + + if (node < 0) { + WD_ERR("invalid: failed to get numa node!\n"); + return (handle_t)(-WD_EINVAL); + } + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return (handle_t)(-WD_EINVAL); + } + + skey = malloc(sizeof(struct sched_key)); + if (!skey) { + WD_ERR("failed to alloc memory for session sched key!\n"); + return (handle_t)(-WD_ENOMEM); + } + + if (!param) { + memset(skey, 0, sizeof(struct sched_key)); + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; + skey->ctx_prop = UADK_CTX_HW; + WD_INFO("loop don't set scheduler parameters!\n"); + } else if (param->numa_id < 0) { + skey->type = param->type; + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; + skey->ctx_prop = param->ctx_prop; + } else { + skey->type = param->type; + skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; + } + + //if (skey->numa_id < 0) { + // WD_ERR("failed to get valid sched numa region!\n"); + // goto out; + //} + skey->numa_id = 0; + + skey->sync_ctxid = loop_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); + skey->async_ctxid = loop_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); + if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { + WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); + goto out; + } + WD_ERR("sync_ctxid is: %u; async_ctxid is: %u!\n", skey->sync_ctxid, skey->async_ctxid); + ctx_prop = skey->ctx_prop; + skey->ctx_prop = UADK_CTX_CE_INS; + skey->sw_sync_ctxid = loop_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); + skey->sw_async_ctxid = loop_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); + skey->ctx_prop = ctx_prop; + + WD_ERR("fb ctxid is: %u, %u!\n", skey->sw_sync_ctxid, skey->sw_async_ctxid); + + return (handle_t)skey; + +out: + free(skey); + return (handle_t)(-WD_EINVAL); +} + +/* + * loop_sched_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. + * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. + * @sched_key: The key of schedule region. + * @sched_mode: The sched async/sync mode. + * + * The user must init the schedule info through session_sched_init + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_key *key = (struct sched_key *)sched_key; + + if (unlikely(!h_sched_ctx || !key)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (key->sw_sync_ctxid == INVALID_POS || key->sw_async_ctxid == INVALID_POS) + return session_sched_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); + + if (sched_mode == CTX_MODE_SYNC) { + if (sched_ctx->balancer.switch_slice == LOOP_SWITH_TIME) { + sched_ctx->balancer.switch_slice = 0; + sched_ctx->balancer.hw_dfx_num++; + /* run in HW */ + return key->sync_ctxid; + } else { + sched_ctx->balancer.switch_slice++; + /* run in soft CE */ + sched_ctx->balancer.sw_dfx_num++; + return key->sw_sync_ctxid; + } + } else { // Async mode + if (sched_ctx->balancer.hw_task_num > sched_ctx->balancer.sw_task_num) + sched_ctx->balancer.next_send_type = SCHED_SEND_SW; + else + sched_ctx->balancer.next_send_type = SCHED_SEND_HW; + + if (sched_ctx->balancer.next_send_type == SCHED_SEND_HW) { + /* run in HW */ + sched_ctx->balancer.hw_task_num++; + sched_ctx->balancer.hw_dfx_num++; + return key->async_ctxid; + } else { + /* run in soft CE */ + sched_ctx->balancer.sw_task_num++; + sched_ctx->balancer.sw_dfx_num++; + return key->sw_async_ctxid; + } + } +} + +static int loop_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, + __u32 expect, __u32 *count) +{ + struct sched_ctx_region **region; + bool region_valid = false; + __u32 begin, end; + __u32 i, j; + int ret; + + for (j = 0; j < SCHED_REGION_NUM; j++) { + switch (j) { + case 0: + region = sched_ctx->sched_info[numa_id].ctx_region; + region_valid = sched_ctx->sched_info[numa_id].hw_valid; + break; + case 1: + region = sched_ctx->sched_info[numa_id].ce_ctx_region; + region_valid = sched_ctx->sched_info[numa_id].ce_valid; + break; + case 2: + region = sched_ctx->sched_info[numa_id].sve_ctx_region; + region_valid = sched_ctx->sched_info[numa_id].sve_valid; + break; + case 3: + region = sched_ctx->sched_info[numa_id].soft_ctx_region; + region_valid = sched_ctx->sched_info[numa_id].soft_valid; + break; + } + + if (!region_valid) + continue; + + for (i = 0; i < sched_ctx->type_num; i++) { + if (!region[SCHED_MODE_ASYNC][i].valid) + continue; + + begin = region[SCHED_MODE_ASYNC][i].begin; + end = region[SCHED_MODE_ASYNC][i].end; + //WD_ERR("session_poll_policy_rr from %u ---> %u!\n", begin, end); + ret = session_poll_region(sched_ctx, begin, end, expect, count); + if (unlikely(ret)) + return ret; + } + + if (j == 0) { + sched_ctx->balancer.hw_task_num -= *count; + } else { + sched_ctx->balancer.sw_task_num -= *count; + } + } + + return 0; +} + +/* + * loop_poll_policy - The polling policy matches the pick next ctx. + * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. + * @cfg: The global resoure info. + * @expect: User expect poll msg num. + * @count: The actually poll num. + * + * The user must init the schedule info through wd_sched_rr_instance, the + * func interval will not check the valid, becouse it will affect performance. + */ +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_info *sched_info; + __u32 loop_time = 0; + __u32 last_count = 0; + __u16 i; + int ret; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { + WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); + return -WD_EINVAL; + } + + sched_info = sched_ctx->sched_info; + + /* + * Try different numa's ctx if we can't receive any + * package last time, it is more efficient. In most + * bad situation, poll ends after MAX_POLL_TIMES loop. + */ + while (++loop_time < MAX_POLL_TIMES) { + for (i = 0; i < sched_ctx->numa_num;) { + /* If current numa is not valid, find next. */ + if (!sched_info[i].nm_valid) { + i++; + continue; + } + + last_count = *count; + ret = loop_poll_policy_rr(sched_ctx, i, expect, count); + if (unlikely(ret)) + return ret; + + if (expect == *count) + return 0; + + /* + * If no package is received, find next numa, + * otherwise, keep receiving packets at this node. + */ + if (last_count == *count) + i++; + } + } + + return 0; +} + +static handle_t loop_sched_rte_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + int cpu = sched_getcpu(); + int node = numa_node_of_cpu(cpu); + struct sched_key *skey; + int ret; + + if (node < 0) { + WD_ERR("invalid: failed to get numa node!\n"); + return (handle_t)(-WD_EINVAL); + } + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return (handle_t)(-WD_EINVAL); + } + + skey = malloc(sizeof(struct sched_key)); + if (!skey) { + WD_ERR("failed to alloc memory for session sched key!\n"); + return (handle_t)(-WD_ENOMEM); + } + + if (!param) { + memset(skey, 0, sizeof(struct sched_key)); + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; + skey->ctx_prop = UADK_CTX_HW; + WD_INFO("loop don't set scheduler parameters!\n"); + } else if (param->numa_id < 0) { + skey->type = param->type; + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; + skey->ctx_prop = param->ctx_prop; + } else { + skey->type = param->type; + skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; + } + + //if (skey->numa_id < 0) { + // WD_ERR("failed to get valid sched numa region!\n"); + // goto out; + //} + skey->numa_id = 0; + skey->sync_ctxid = INVALID_POS; + skey->async_ctxid = INVALID_POS; + skey->sw_sync_ctxid = INVALID_POS; + skey->sw_async_ctxid = INVALID_POS; + + ret = sched_key_valid(sched_ctx, skey); + if (!ret) + goto out; + + return (handle_t)skey; +out: + free(skey); + return (handle_t)(-WD_EINVAL); +} + +static __u32 loop_sched_rte_pick_ctx(struct wd_sched_ctx *sched_ctx, + struct sched_key *key, const int sched_mode) +{ + struct sched_ctx_region *region = NULL; + + key->mode = sched_mode; + region = loop_get_ctx_range(sched_ctx, key); + if (!region) + return INVALID_POS; + + return sched_get_next_pos_rr(region, NULL); +} + +static __u32 loop_sched_rte_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_key *skey = (struct sched_key *)sched_key; + __u32 rte_ctxid = INVALID_POS; + + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_ctx->balancer.switch_slice == LOOP_SWITH_TIME) { + sched_ctx->balancer.switch_slice = 0; + skey->ctx_prop = UADK_CTX_HW; + /* run in HW */ + if (sched_mode == CTX_MODE_SYNC) { + if (skey->sync_ctxid != INVALID_POS) + rte_ctxid = skey->sync_ctxid; + else { + rte_ctxid = loop_sched_rte_pick_ctx(sched_ctx, skey, CTX_MODE_SYNC); + skey->sync_ctxid = rte_ctxid; + } + } else { + if (skey->async_ctxid != INVALID_POS) + rte_ctxid = skey->async_ctxid; + else { + rte_ctxid = loop_sched_rte_pick_ctx(sched_ctx, skey, CTX_MODE_ASYNC); + skey->async_ctxid = rte_ctxid; + } + } + } else { + sched_ctx->balancer.switch_slice++; + skey->ctx_prop = UADK_CTX_CE_INS; + /* run in soft CE */ + if (sched_mode == CTX_MODE_SYNC) { + if (skey->sw_sync_ctxid != INVALID_POS) + rte_ctxid = skey->sw_sync_ctxid; + else { + rte_ctxid = loop_sched_rte_pick_ctx(sched_ctx, skey, CTX_MODE_SYNC); + skey->sw_sync_ctxid = rte_ctxid; + } + } else { + if (skey->sw_async_ctxid != INVALID_POS) + rte_ctxid = skey->sw_async_ctxid; + else { + rte_ctxid = loop_sched_rte_pick_ctx(sched_ctx, skey, CTX_MODE_ASYNC); + skey->sw_async_ctxid = rte_ctxid; + } + } + } + + return rte_ctxid; +} + static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", @@ -463,7 +931,19 @@ static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, - } + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + }, { + .name = "Loop rte scheduler", + .sched_policy = SCHED_POLICY_RTE_LOOP, + .sched_init = loop_sched_rte_init, + .pick_next_ctx = loop_sched_rte_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + }, };

static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) @@ -473,7 +953,7 @@ static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int nod int i, tmp;

for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { + if (sched_info[i].nm_valid) { tmp = numa_distance(node, i); if (dis > tmp) { valid_id = i; @@ -492,7 +972,7 @@ static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) int *numa_map = sched_ctx->numa_map;

for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) + if (sched_info[i].nm_valid) numa_map[i] = i; else numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); @@ -504,7 +984,7 @@ int wd_sched_rr_instance(const struct wd_sched *sched, { struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; + __u8 type, mode, prop; int numa_id;

if (!sched || !sched->h_sched_ctx || !param) { @@ -540,6 +1020,12 @@ int wd_sched_rr_instance(const struct wd_sched *sched, return -WD_EINVAL; }

+ prop = param->ctx_prop; + if (prop > UADK_CTX_SOFT) { + WD_ERR("invalid: sched_ctx's prop is %u\n", prop); + return -WD_EINVAL; + } + sched_info = sched_ctx->sched_info;

if (!sched_info[numa_id].ctx_region[mode]) { @@ -548,16 +1034,52 @@ int wd_sched_rr_instance(const struct wd_sched *sched, return -WD_EINVAL; }

- sched_info[numa_id].ctx_region[mode][type].begin = param->begin; - sched_info[numa_id].ctx_region[mode][type].end = param->end; - sched_info[numa_id].ctx_region[mode][type].last = param->begin; - sched_info[numa_id].ctx_region[mode][type].valid = true; - sched_info[numa_id].valid = true; + WD_ERR("instance uadk ctx: numa id: %u, mode: %u, type: %u!\n", numa_id, mode, type);

- wd_sched_map_cpus_to_dev(sched_ctx); - - pthread_mutex_init(&sched_info[numa_id].ctx_region[mode][type].lock, + switch (prop) { + case UADK_CTX_HW: + sched_info[numa_id].ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].ctx_region[mode][type].end = param->end; + sched_info[numa_id].ctx_region[mode][type].last = param->begin; + sched_info[numa_id].ctx_region[mode][type].valid = true; + sched_info[numa_id].hw_valid = true; + pthread_mutex_init(&sched_info[numa_id].ctx_region[mode][type].lock, + NULL); + WD_ERR("instance HW ctx: begin: %u ----> end: %u!\n", param->begin, param->end); + break; + case UADK_CTX_CE_INS: + sched_info[numa_id].ce_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].ce_ctx_region[mode][type].end = param->end; + sched_info[numa_id].ce_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].ce_ctx_region[mode][type].valid = true; + sched_info[numa_id].ce_valid = true; + pthread_mutex_init(&sched_info[numa_id].ce_ctx_region[mode][type].lock, + NULL); + WD_ERR("instance CE ctx: begin: %u ----> end: %u!\n", param->begin, param->end); + break; + case UADK_CTX_SVE_INS: + sched_info[numa_id].sve_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].sve_ctx_region[mode][type].end = param->end; + sched_info[numa_id].sve_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].sve_ctx_region[mode][type].valid = true; + sched_info[numa_id].sve_valid = true; + pthread_mutex_init(&sched_info[numa_id].sve_ctx_region[mode][type].lock, + NULL); + WD_ERR("instance SVE ctx: begin: %u ----> end: %u!\n", param->begin, param->end); + break; + case UADK_CTX_SOFT: + sched_info[numa_id].soft_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].soft_ctx_region[mode][type].end = param->end; + sched_info[numa_id].soft_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].soft_ctx_region[mode][type].valid = true; + sched_info[numa_id].soft_valid = true; + pthread_mutex_init(&sched_info[numa_id].soft_ctx_region[mode][type].lock, NULL); + WD_ERR("instance Soft ctx: begin: %u ----> end: %u!\n", param->begin, param->end); + break; + } + sched_info[numa_id].nm_valid = true; + wd_sched_map_cpus_to_dev(sched_ctx);

return 0; } @@ -585,9 +1107,28 @@ void wd_sched_rr_release(struct wd_sched *sched) free(sched_info[i].ctx_region[j]); sched_info[i].ctx_region[j] = NULL; } + + if (sched_info[i].ce_ctx_region[j]) { + free(sched_info[i].ce_ctx_region[j]); + sched_info[i].ce_ctx_region[j] = NULL; + } + + if (sched_info[i].sve_ctx_region[j]) { + free(sched_info[i].sve_ctx_region[j]); + sched_info[i].sve_ctx_region[j] = NULL; + } + + if (sched_info[i].soft_ctx_region[j]) { + free(sched_info[i].soft_ctx_region[j]); + sched_info[i].soft_ctx_region[j] = NULL; + } } }

+ /* Release sched dfx info */ + WD_ERR("scheduler balance hw task num: %u, sw task num: %u\n", + sched_ctx->balancer.hw_dfx_num, sched_ctx->balancer.sw_dfx_num); + info_out: free(sched_ctx); ctx_out: @@ -614,13 +1155,79 @@ static int numa_num_check(__u16 numa_num) return 0; }

+static int wd_sched_region_init(struct wd_sched_ctx *sched_ctx, + __u8 type_num, __u16 numa_num) +{ + struct wd_sched_info *sched_info = sched_ctx->sched_info; + int i, j; + int ri, idx; + + for (i = 0; i < numa_num; i++) { + for (j = 0; j < SCHED_MODE_BUTT; j++) { + sched_info[i].ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].ctx_region[j]) + goto hw_err; + + sched_info[i].ce_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].ce_ctx_region[j]) + goto ce_err; + + sched_info[i].sve_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].sve_ctx_region[j]) + goto sve_err; + + sched_info[i].soft_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].soft_ctx_region[j]) + goto soft_err; + } + sched_info[i].nm_valid = false; + sched_info[i].hw_valid = false; + sched_info[i].ce_valid = false; + sched_info[i].sve_valid = false; + sched_info[i].soft_valid = false; + } + + return 0; + +soft_err: + free(sched_info[i].sve_ctx_region[j]); + sched_info[i].sve_ctx_region[j] = NULL; +sve_err: + free(sched_info[i].ce_ctx_region[j]); + sched_info[i].ce_ctx_region[j] = NULL; +ce_err: + free(sched_info[i].ctx_region[j]); + sched_info[i].ctx_region[j] = NULL; +hw_err: + for (ri = i - 1; ri >= 0; ri--) { + for (idx = 0; idx < SCHED_MODE_BUTT; idx++) { + free(sched_info[ri].ctx_region[idx]); + sched_info[ri].ctx_region[idx] = NULL; + + free(sched_info[ri].ce_ctx_region[idx]); + sched_info[ri].ce_ctx_region[idx] = NULL; + + free(sched_info[ri].sve_ctx_region[idx]); + sched_info[ri].sve_ctx_region[idx] = NULL; + + free(sched_info[ri].soft_ctx_region[idx]); + sched_info[ri].soft_ctx_region[idx] = NULL; + } + } + + return -WD_EINVAL; +} + struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, __u16 numa_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int i, j; + int ret;

if (numa_num_check(numa_num)) return NULL; @@ -650,15 +1257,9 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, sched_type == SCHED_POLICY_SINGLE) goto simple_ok;

- sched_info = sched_ctx->sched_info; - for (i = 0; i < numa_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } - } + ret = wd_sched_region_init(sched_ctx, type_num, numa_num); + if (ret) + goto err_out;

simple_ok: sched_ctx->poll_func = func;

Add heterogeneous computing functions to the soft and hard computing drivers of UADK. Adapt the drivers to ensure that different devices can perform heterogeneous computing at the same time and provide acceleration functions.

In order to ensure normal compilation, some drivers have been processed with hac mode, and can be compiled directly through UADK_MK.SH

Signed-off-by: Longfang Liu liulongfang@huawei.com --- Makefile.am | 139 +++++++++++------------------------- drv/hash_mb/hash_mb.c | 8 +++ drv/hash_mb/hash_mb.h | 3 + drv/hisi_comp.c | 2 + drv/hisi_sec.c | 99 ++++++++++++------------- drv/isa_ce_sm3.c | 25 +++++++ drv/isa_ce_sm3.h | 3 + drv/isa_ce_sm4.c | 28 +++++++- drv/isa_ce_sm4.h | 5 +- include/drv/wd_comp_drv.h | 3 +- include/drv/wd_digest_drv.h | 3 +- uadk_mk.sh | 9 +++ 12 files changed, 170 insertions(+), 157 deletions(-) create mode 100644 uadk_mk.sh

diff --git a/Makefile.am b/Makefile.am index ba3308a..d9604b1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,9 +1,7 @@ ACLOCAL_AMFLAGS = -I m4 -I./include AUTOMAKE_OPTIONS = foreign subdir-objects -AM_CFLAGS=-Wall -Werror -fno-strict-aliasing -I$(top_srcdir)/include -AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -D_FORTIFY_SOURCE=2 \ - -O2 -ftrapv -Wl,-z,relro,-z,now -Wl,-s -CLEANFILES = +AM_CFLAGS = -std=gnu11 -Wall -Werror -Wextra -Wno-unused-parameter -Wfloat-equal -I$(top_srcdir)/include +AM_CFLAGS += -g -O0 -fno-omit-frame-pointer -fPIC -fPIE -pie -fstack-protector-strong -ftrapv

if WITH_LOG_FILE AM_CFLAGS+=-DWITH_LOG_FILE="$(with_log_file)" @@ -22,77 +20,40 @@ endif # WITH_LOG_FILE # y = minor # z = revision MAJOR = 2 -MINOR = 7 -REVISION = 0 +MINOR = 3 +REVISION = 22 UADK_VERSION = -version-number ${MAJOR}:${MINOR}:${REVISION}

-DAY = 6 -MONTH = June -YEAR = 2024 +DAY = 7 +MONTH = Mar +YEAR = 2022 AM_CFLAGS+= -DUADK_VERSION_NUMBER=""UADK version: ${MAJOR}.${MINOR}.${REVISION}"" AM_CFLAGS+= -DUADK_RELEASED_TIME=""Released ${MONTH} ${DAY}, ${YEAR}""

-pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \ - include/wd_comp.h include/wd_dh.h include/wd_digest.h \ - include/wd_rsa.h include/uacce.h include/wd_alg_common.h \ - include/wd_ecc.h include/wd_sched.h include/wd_alg.h \ - include/wd_zlibwrapper.h include/wd_dae.h include/wd_agg.h - -nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \ - v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h - -lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libwd_dae.la - -uadk_driversdir=$(libdir)/uadk -uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ - libisa_ce.la libisa_sve.la libhisi_dae.la - -libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ - v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ - v1/wd_rng.c v1/wd_rng.h \ - v1/wd_rsa.c v1/wd_rsa.h \ - v1/wd_aead.c v1/wd_aead.h \ - v1/wd_dh.c v1/wd_dh.h \ - v1/wd_comp.c v1/wd_comp.h \ - v1/wd_cipher.c v1/wd_cipher.h \ - v1/wd_digest.c v1/wd_digest.h \ - v1/wd_util.c v1/wd_util.h \ - v1/wd_bmm.c v1/wd_bmm.h \ - v1/wd_ecc.c v1/wd_ecc.h \ - v1/wd_sgl.c v1/wd_sgl.h \ - v1/drv/hisi_qm_udrv.c v1/drv/hisi_qm_udrv.h \ - v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ - v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ - v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ - v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h - -libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ - wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h +include_HEADERS = include/wd.h include/wd_cipher.h include/wd_comp.h \ + include/uacce.h include/wd_alg_common.h \ + include/wd_sched.h include/wd_alg.h + +lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libhisi_zip.la \ + libisa_ce.la libisa_sve.la libhisi_sec.la + +libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h

libwd_comp_la_SOURCES=wd_comp.c wd_comp.h wd_comp_drv.h wd_util.c wd_util.h \ - wd_sched.c wd_sched.h wd.c wd.h wd_zlibwrapper.c + wd_sched.c wd_sched.h wd.c wd.h \ + wd_zlibwrapper.c wd_zlibwrapper.h

libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ hisi_qm_udrv.h wd_comp_drv.h

libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ - wd_aead.c wd_aead.h wd_aead_drv.h \ - wd_rsa.c wd_rsa.h wd_rsa_drv.h \ - wd_dh.c wd_dh.h wd_dh_drv.h \ - wd_ecc.c wd_ecc.h wd_ecc_drv.h \ wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ wd_sched.c wd_sched.h \ wd.c wd.h

-libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ - lib/crypto/aes.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h - -libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h - -libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ +libisa_ce_la_SOURCES=arm_arch_ce.h wd_cipher_drv.h drv/isa_ce_sm3.c \ + drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h

libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ @@ -101,75 +62,55 @@ libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb. drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S

-libhisi_dae_la_SOURCES=drv/hisi_dae.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h +libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ + hisi_qm_udrv.h wd_cipher_drv.h \ + lib/crypto/aes.c lib/crypto/galois.c aes.h galois.h

if WD_STATIC_DRV -AM_CFLAGS += -DWD_STATIC_DRV -fPIC +AM_CFLAGS += -DWD_STATIC_DRV AM_CFLAGS += -DWD_NO_LOG

-libwd_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma +libwd_la_LIBADD = $(libwd_la_OBJECTS) -lnuma -lpthread

-libwd_comp_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma +libwd_comp_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lpthread -lnuma libwd_comp_la_DEPENDENCIES = libwd.la

-libhisi_zip_la_LIBADD = -ldl +libhisi_zip_la_LIBADD = -ldl -lpthread

-libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma +libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread libwd_crypto_la_DEPENDENCIES = libwd.la

-libwd_dae_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -libwd_dae_la_DEPENDENCIES = libwd.la - -libhisi_sec_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) -libhisi_sec_la_DEPENDENCIES = libwd.la libwd_crypto.la - -libhisi_hpre_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) -libhisi_hpre_la_DEPENDENCIES = libwd.la libwd_crypto.la - libisa_ce_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) libisa_ce_la_DEPENDENCIES = libwd.la libwd_crypto.la

libisa_sve_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) libisa_sve_la_DEPENDENCIES = libwd.la libwd_crypto.la

-libhisi_dae_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_dae_la_OBJECTS) -libhisi_dae_la_DEPENDENCIES = libwd.la libwd_dae.la +libhisi_sec_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) $(libisa_ce_la_OBJECTS) $(libisa_sve_la_OBJECTS) +libhisi_sec_la_DEPENDENCIES = libwd.la libwd_crypto.la libisa_ce.la libisa_sve.la

else UADK_WD_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd.map UADK_CRYPTO_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_crypto.map -UADK_DAE_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_dae.map +#UADK_DAE_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_dae.map UADK_COMP_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_comp.map UADK_V1_SYMBOL= -Wl,--version-script,$(top_srcdir)/v1/libwd.map

libwd_la_LDFLAGS=$(UADK_VERSION) $(UADK_WD_SYMBOL) $(UADK_V1_SYMBOL) -libwd_la_LIBADD= -ldl -lnuma +libwd_la_LIBADD= -lnuma

-libwd_comp_la_LIBADD= -lwd -ldl -lnuma -libwd_comp_la_LDFLAGS=$(UADK_VERSION) $(UADK_COMP_SYMBOL) -lpthread +libwd_comp_la_LIBADD= -lwd -ldl -lpthread -lnuma +libwd_comp_la_LDFLAGS=$(UADK_VERSION) $(UADK_COMP_SYMBOL) libwd_comp_la_DEPENDENCIES= libwd.la

-libwd_crypto_la_LIBADD= -lwd -ldl -lnuma -libwd_crypto_la_LDFLAGS=$(UADK_VERSION) $(UADK_CRYPTO_SYMBOL) -lpthread +libwd_crypto_la_LIBADD= -lwd -ldl -lnuma -lm -lpthread +libwd_crypto_la_LDFLAGS=$(UADK_VERSION) $(UADK_CRYPTO_SYMBOL) libwd_crypto_la_DEPENDENCIES= libwd.la

-libwd_dae_la_LIBADD= -lwd -ldl -lnuma -libwd_dae_la_LDFLAGS=$(UADK_VERSION) $(UADK_DAE_SYMBOL) -libwd_dae_la_DEPENDENCIES= libwd.la - libhisi_zip_la_LIBADD= -lwd -ldl -lwd_comp libhisi_zip_la_LDFLAGS=$(UADK_VERSION) libhisi_zip_la_DEPENDENCIES= libwd.la libwd_comp.la

-libhisi_sec_la_LIBADD= -lwd -lwd_crypto -libhisi_sec_la_LDFLAGS=$(UADK_VERSION) -libhisi_sec_la_DEPENDENCIES= libwd.la libwd_crypto.la - -libhisi_hpre_la_LIBADD= -lwd -lwd_crypto -libhisi_hpre_la_LDFLAGS=$(UADK_VERSION) -libhisi_hpre_la_DEPENDENCIES= libwd.la libwd_crypto.la - libisa_ce_la_LIBADD= -lwd -lwd_crypto libisa_ce_la_LDFLAGS=$(UADK_VERSION) libisa_ce_la_DEPENDENCIES= libwd.la libwd_crypto.la @@ -178,14 +119,14 @@ libisa_sve_la_LIBADD= -lwd -lwd_crypto libisa_sve_la_LDFLAGS=$(UADK_VERSION) libisa_sve_la_DEPENDENCIES= libwd.la libwd_crypto.la

-libhisi_dae_la_LIBADD= -lwd -lwd_dae -libhisi_dae_la_LDFLAGS=$(UADK_VERSION) -libhisi_dae_la_DEPENDENCIES= libwd.la libwd_dae.la +libhisi_sec_la_LIBADD= -lwd -lwd_crypto +libhisi_sec_la_LDFLAGS=$(UADK_VERSION) +libhisi_sec_la_DEPENDENCIES= libwd.la libwd_crypto.la

endif # WD_STATIC_DRV

pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = lib/libwd_crypto.pc lib/libwd_comp.pc lib/libwd.pc -CLEANFILES += $(pkgconfig_DATA)

-SUBDIRS=. test v1/test uadk_tool sample +#SUBDIRS=. test v1/test uadk_tool sample +SUBDIRS=. uadk_tool \ No newline at end of file diff --git a/drv/hash_mb/hash_mb.c b/drv/hash_mb/hash_mb.c index 3f5b1a5..27a06a5 100644 --- a/drv/hash_mb/hash_mb.c +++ b/drv/hash_mb/hash_mb.c @@ -818,7 +818,11 @@ static struct wd_alg_driver hash_mb_driver[] = { GEN_HASH_ALG_DRIVER("md5"), };

+#ifdef WD_STATIC_DRV +void hash_mb_probe(void) +#else static void __attribute__((constructor)) hash_mb_probe(void) +#endif { unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); @@ -837,7 +841,11 @@ static void __attribute__((constructor)) hash_mb_probe(void) } }

+#ifdef WD_STATIC_DRV +void hash_mb_remove(void) +#else static void __attribute__((destructor)) hash_mb_remove(void) +#endif { unsigned long auxval = getauxval(AT_HWCAP); size_t alg_num = ARRAY_SIZE(hash_mb_driver); diff --git a/drv/hash_mb/hash_mb.h b/drv/hash_mb/hash_mb.h index aba5ec9..b81a819 100644 --- a/drv/hash_mb/hash_mb.h +++ b/drv/hash_mb/hash_mb.h @@ -54,6 +54,9 @@ void md5_mb_asimd_x4(struct hash_job *job1, struct hash_job *job2, void md5_mb_asimd_x1(struct hash_job *job, int len); int md5_mb_sve_max_lanes(void);

+void hash_mb_probe(void); +void hash_mb_remove(void); + #ifdef __cplusplus } #endif diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index ffe8eac..62571e6 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -803,6 +803,8 @@ static int hisi_zip_init(void *conf, void *priv) memcpy(&zip_ctx->config, config, sizeof(struct wd_ctx_config_internal)); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW) + continue; h_ctx = config->ctxs[i].ctx; qm_priv.sqe_size = sizeof(struct hisi_zip_sqe); qm_priv.op_type = config->ctxs[i].op_type; diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index f008c16..b33b64a 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -11,6 +11,10 @@ #include "wd_digest.h" #include "wd_aead.h"

+#include "isa_ce_sm4.h" +#include "isa_ce_sm3.h" +#include "hash_mb/hash_mb.h" + #define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 #define WORD_ALIGNMENT_MASK 0x3 @@ -1567,7 +1571,7 @@ static int fill_digest_long_hash(handle_t h_qp, struct wd_digest_msg *msg, static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_digest_msg *recv_msg) { - struct wd_digest_msg *temp_msg; + struct wd_digest_msg *temp_msg = NULL; __u16 done;

done = sqe->type2.done_flag & SEC_DONE_MASK; @@ -2002,7 +2006,7 @@ put_sgl: static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, struct wd_digest_msg *recv_msg) { - struct wd_digest_msg *temp_msg; + struct wd_digest_msg *temp_msg = NULL; __u16 done;

done = sqe->done_flag & SEC_DONE_MASK; @@ -2260,9 +2264,9 @@ static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) } }

- if (unlikely((__u64)msg->in_bytes + msg->assoc_bytes > MAX_INPUT_DATA_LEN)) { - WD_ERR("aead input data length is too long, size = %llu\n", - (__u64)msg->in_bytes + msg->assoc_bytes); + if (unlikely(msg->in_bytes + msg->assoc_bytes > MAX_INPUT_DATA_LEN)) { + WD_ERR("aead input data length is too long, size = %u\n", + msg->in_bytes + msg->assoc_bytes); return -WD_EINVAL; }

@@ -2521,11 +2525,6 @@ int aead_msg_state_check(struct wd_aead_msg *msg) } }

- if (unlikely(msg->msg_state != AEAD_MSG_BLOCK && msg->data_fmt == WD_SGL_BUF)) { - WD_ERR("invalid: sgl mode not supports stream mode!\n"); - return -WD_EINVAL; - } - return 0; }

@@ -2565,12 +2564,10 @@ static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) fill_aead_bd2_addr(msg, &sqe);

ret = fill_stream_bd2(msg, &sqe); - if (ret == WD_SOFT_COMPUTING) { - ret = 0; - goto put_sgl; - } else if (unlikely(ret)) { - goto put_sgl; - } + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret;

hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag; @@ -2580,16 +2577,14 @@ static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) if (ret != -WD_EBUSY) WD_ERR("aead send sqe is err(%d)!\n", ret);

- goto put_sgl; + if (msg->data_fmt == WD_SGL_BUF) + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, + msg->out); + + return ret; }

return 0; - -put_sgl: - if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - - return ret; }

static void update_stream_counter(struct wd_aead_msg *recv_msg) @@ -2610,7 +2605,7 @@ static void update_stream_counter(struct wd_aead_msg *recv_msg) static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { - struct wd_aead_msg *temp_msg; + struct wd_aead_msg *temp_msg = NULL; __u16 done, icv;

done = sqe->type2.done_flag & SEC_DONE_MASK; @@ -2631,7 +2626,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2643,7 +2638,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, temp_msg = recv_msg; }

- update_stream_counter(temp_msg); + update_stream_counter(recv_msg);

if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); @@ -2652,17 +2647,8 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) { /* Asynchronous mode does not use the sent message, so ignores it */ - if (qp->q_info.qp_mode == CTX_MODE_ASYNC) - return false; - /* - * For aead gcm stream mode, due to some hardware limitations, - * the final message was not sent to hardware if the qm is - * not higher than v3 version or the input length of the - * message is 0, the software calculation has been executed. - */ - if (msg->msg_state == AEAD_MSG_END && msg->cmode == WD_CIPHER_GCM && - (qp->q_info.hw_type <= HISI_QM_API_VER3_BASE || !msg->in_bytes)) - return true; + if (msg->cmode == WD_CIPHER_GCM) + return (msg->msg_state == AEAD_MSG_END) && qp->q_info.qp_mode == CTX_MODE_SYNC;

return false; } @@ -2972,12 +2958,10 @@ static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) fill_aead_bd3_addr(msg, &sqe);

ret = fill_stream_bd3(h_qp, msg, &sqe); - if (ret == WD_SOFT_COMPUTING) { - ret = 0; - goto put_sgl; - } else if (unlikely(ret)) { - goto put_sgl; - } + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret;

hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = msg->tag; @@ -2986,22 +2970,20 @@ static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) if (ret != -WD_EBUSY) WD_ERR("aead send sqe v3 is err(%d)!\n", ret);

- goto put_sgl; + if (msg->data_fmt == WD_SGL_BUF) + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, + msg->out); + + return ret; }

return 0; - -put_sgl: - if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - - return ret; }

static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, struct wd_aead_msg *recv_msg) { - struct wd_aead_msg *temp_msg; + struct wd_aead_msg *temp_msg = NULL; __u16 done, icv;

done = sqe->done_flag & SEC_DONE_MASK; @@ -3023,7 +3005,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -3035,7 +3017,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, temp_msg = recv_msg; }

- update_stream_counter(temp_msg); + update_stream_counter(recv_msg);

if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); @@ -3083,9 +3065,12 @@ static int hisi_sec_init(void *conf, void *priv) return -WD_EINVAL; }

+ WD_ERR("debug: call function: %s!\n", __func__); qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW) + continue; h_ctx = config->ctxs[i].ctx; /* setting the type is 0 for sqc_type */ qm_priv.op_type = 0; @@ -3164,6 +3149,10 @@ static void __attribute__((constructor)) hisi_sec2_probe(void) WD_ERR("Error: register SEC %s failed!\n", aead_alg_driver[i].alg_name); } + + isa_ce_probe(); + sm3_ce_probe(); + hash_mb_probe(); }

#ifdef WD_STATIC_DRV @@ -3187,5 +3176,9 @@ static void __attribute__((destructor)) hisi_sec2_remove(void) alg_num = ARRAY_SIZE(aead_alg_driver); for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&aead_alg_driver[i]); + + isa_ce_remove(); + sm3_ce_remove(); + hash_mb_remove(); }

diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index c8812df..fa0256c 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -44,7 +44,11 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .get_usage = sm3_ce_get_usage, };

+#ifdef WD_STATIC_DRV +void sm3_ce_probe(void) +#else static void __attribute__((constructor)) sm3_ce_probe(void) +#endif { int ret;

@@ -54,7 +58,11 @@ static void __attribute__((constructor)) sm3_ce_probe(void) WD_ERR("Error: register SM3 CE failed!\n"); }

+#ifdef WD_STATIC_DRV +void sm3_ce_remove(void) +#else static void __attribute__((destructor)) sm3_ce_remove(void) +#endif { wd_alg_driver_unregister(&sm3_ce_alg_driver); } @@ -340,6 +348,7 @@ static int do_hmac_sm3_ce(struct wd_digest_msg *msg, __u8 *out_hmac)

static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; __u8 digest[SM3_DIGEST_SIZE] = {0}; int ret; @@ -349,6 +358,10 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) return -WD_EINVAL; }

+ ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM3 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -363,11 +376,23 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) ret = -WD_EINVAL; }

+ ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; }

static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return WD_SUCCESS; }

diff --git a/drv/isa_ce_sm3.h b/drv/isa_ce_sm3.h index 13edb0a..b3d8810 100644 --- a/drv/isa_ce_sm3.h +++ b/drv/isa_ce_sm3.h @@ -79,6 +79,9 @@ struct sm3_ce_drv_ctx { void sm3_ce_block_compress(__u32 word_reg[SM3_STATE_WORDS], const __u8 *src, size_t blocks);

+void sm3_ce_probe(void); +void sm3_ce_remove(void); + #ifdef __cplusplus } #endif diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 6475246..1f54e1f 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -11,6 +11,7 @@ * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */

+#include "wd_alg.h" #include "drv/wd_cipher_drv.h" #include "isa_ce_sm4.h" #include "wd_cipher.h" @@ -323,15 +324,20 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey

static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; int ret = 0;

- if (!msg) { + if (!msg || !ctx) { WD_ERR("invalid: input sm4 msg is NULL!\n"); return -WD_EINVAL; }

+ ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM4 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -384,11 +390,23 @@ static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) return -WD_EINVAL; }

+ ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; }

static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_cipher_msg *msg = wd_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return 0; }

@@ -428,7 +446,11 @@ static struct wd_alg_driver cipher_alg_driver[] = { GEN_CE_ALG_DRIVER("ecb(sm4)", cipher), };

+#ifdef WD_STATIC_DRV +void isa_ce_probe(void) +#else static void __attribute__((constructor)) isa_ce_probe(void) +#endif { __u32 alg_num, i; int ret; @@ -444,7 +466,11 @@ static void __attribute__((constructor)) isa_ce_probe(void) } }

+#ifdef WD_STATIC_DRV +void isa_ce_remove(void) +#else static void __attribute__((destructor)) isa_ce_remove(void) +#endif { __u32 alg_num, i;

diff --git a/drv/isa_ce_sm4.h b/drv/isa_ce_sm4.h index 308619e..d9ce128 100644 --- a/drv/isa_ce_sm4.h +++ b/drv/isa_ce_sm4.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */

#ifndef __SM4_CE_DRV_H #define __SM4_CE_DRV_H @@ -55,6 +55,9 @@ int sm4_v8_xts_decrypt(const unsigned char *in, unsigned char *out, size_t lengt const struct SM4_KEY *key, unsigned char *ivec, const struct SM4_KEY *key2);

+void isa_ce_probe(void); +void isa_ce_remove(void); + #ifdef __cplusplus } #endif diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d..2b95716 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -6,9 +6,8 @@

#include <pthread.h> #include <asm/types.h> - +#include "../wd_alg_common.h" #include "../wd_comp.h" -#include "../wd_util.h"

#ifdef __cplusplus extern "C" { diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index a55ef5b..5e3e821 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -3,8 +3,9 @@ #ifndef __WD_DIGEST_DRV_H #define __WD_DIGEST_DRV_H

+#include <asm/types.h> + #include "../wd_digest.h" -#include "../wd_util.h"

#ifdef __cplusplus extern "C" { diff --git a/uadk_mk.sh b/uadk_mk.sh new file mode 100644 index 0000000..c69937c --- /dev/null +++ b/uadk_mk.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +make clean + +./autogen.sh + +./conf.sh --static + +make -j8