Unify the software ctx and hardware ctx in uadk and merge them on the scheduler. Realize the function of software and hardware calculation together

Signed-off-by: Longfang Liu liulongfang@huawei.com --- Makefile.am | 124 +++++----- drv/hisi_sec.c | 20 +- drv/isa_ce_sm4.c | 9 + drv/isa_ce_sm4.h | 5 + include/wd_alg.h | 1 + include/wd_alg_common.h | 12 + include/wd_sched.h | 1 + include/wd_util.h | 15 +- module.mk | 51 ++-- wd.c | 1 - wd_cipher.c | 95 ++++---- wd_comp.c | 85 +++---- wd_sched.c | 155 ++++++++++-- wd_util.c | 527 +++++++++++++++++++++++++++++----------- 14 files changed, 739 insertions(+), 362 deletions(-)

diff --git a/Makefile.am b/Makefile.am index 6eb8fc3b..d6a0a40c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,36 +32,31 @@ AM_CFLAGS+= -DUADK_VERSION_NUMBER=""UADK version: ${MAJOR}.${MINOR}.${REVISION} AM_CFLAGS+= -DUADK_RELEASED_TIME=""Released ${MONTH} ${DAY}, ${YEAR}""

include_HEADERS = include/wd.h include/wd_cipher.h include/wd_comp.h \ - include/wd_dh.h include/wd_digest.h include/wd_rsa.h \ include/uacce.h include/wd_alg_common.h \ - include/wd_ecc.h include/wd_sched.h include/wd_alg.h \ - include/wd_dae.h include/wd_agg.h - -nobase_include_HEADERS = v1/wd.h v1/wd_cipher.h v1/uacce.h v1/wd_dh.h v1/wd_digest.h \ - v1/wd_rsa.h v1/wd_bmm.h - -lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libhisi_zip.la \ - libhisi_hpre.la libhisi_sec.la libisa_ce.la libisa_sve.la \ - libwd_dae.la libhisi_dae.la - -libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ - v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ - v1/wd_rng.c v1/wd_rng.h \ - v1/wd_rsa.c v1/wd_rsa.h \ - v1/wd_aead.c v1/wd_aead.h \ - v1/wd_dh.c v1/wd_dh.h \ - v1/wd_comp.c v1/wd_comp.h \ - v1/wd_cipher.c v1/wd_cipher.h \ - v1/wd_digest.c v1/wd_digest.h \ - v1/wd_util.c v1/wd_util.h \ - v1/wd_bmm.c v1/wd_bmm.h \ - v1/wd_ecc.c v1/wd_ecc.h \ - v1/wd_sgl.c v1/wd_sgl.h \ - v1/drv/hisi_qm_udrv.c v1/drv/hisi_qm_udrv.h \ - v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ - v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ - v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ - v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h + include/wd_sched.h include/wd_alg.h + +lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la \ + libhisi_zip.la libhisi_sec.la \ + libisa_ce.la libisa_sve.la + +libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h +# v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ +# v1/wd_rng.c v1/wd_rng.h \ +# v1/wd_rsa.c v1/wd_rsa.h \ +# v1/wd_aead.c v1/wd_aead.h \ +# v1/wd_dh.c v1/wd_dh.h \ +# v1/wd_comp.c v1/wd_comp.h \ +# v1/wd_cipher.c v1/wd_cipher.h \ +# v1/wd_digest.c v1/wd_digest.h \ +# v1/wd_util.c v1/wd_util.h \ +# v1/wd_bmm.c v1/wd_bmm.h \ +# v1/wd_ecc.c v1/wd_ecc.h \ +# v1/wd_sgl.c v1/wd_sgl.h \ +# v1/drv/hisi_qm_udrv.c v1/drv/hisi_qm_udrv.h \ +# v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ +# v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ +# v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ +# v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h

libwd_comp_la_SOURCES=wd_comp.c wd_comp.h wd_comp_drv.h wd_util.c wd_util.h \ wd_sched.c wd_sched.h wd.c wd.h \ @@ -71,28 +66,30 @@ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ hisi_qm_udrv.h wd_comp_drv.h

libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ - wd_aead.c wd_aead.h wd_aead_drv.h \ - wd_rsa.c wd_rsa.h wd_rsa_drv.h \ - wd_dh.c wd_dh.h wd_dh_drv.h \ - wd_ecc.c wd_ecc.h wd_ecc_drv.h \ - wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ wd_sched.c wd_sched.h \ - wd.c wd.h - -libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ - wd_util.c wd_util.h \ - wd_sched.c wd_sched.h \ - wd.c wd.h + wd.c wd.h \ + arm_arch_ce.h isa_ce_sm3.h isa_ce_sm4.h +# wd_aead.c wd_aead.h wd_aead_drv.h \ +# wd_rsa.c wd_rsa.h wd_rsa_drv.h \ +# wd_dh.c wd_dh.h wd_dh_drv.h \ +# wd_ecc.c wd_ecc.h wd_ecc_drv.h \ +# wd_digest.c wd_digest.h wd_digest_drv.h \ +#libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ +# wd_util.c wd_util.h \ +# wd_sched.c wd_sched.h \ +# wd.c wd.h

libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ + hisi_qm_udrv.h wd_cipher_drv.h \ lib/crypto/aes.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h + aes.h galois.h drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h

-libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h +#libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ +# hisi_qm_udrv.h

-libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ +libisa_ce_la_SOURCES=arm_arch_ce.h wd_cipher_drv.h drv/isa_ce_sm3.c \ + drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h

libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ @@ -101,8 +98,8 @@ libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb. drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S

-libhisi_dae_la_SOURCES=drv/hisi_dae.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h +#libhisi_dae_la_SOURCES=drv/hisi_dae.c drv/hisi_qm_udrv.c \ +# hisi_qm_udrv.h

if WD_STATIC_DRV AM_CFLAGS += -DWD_STATIC_DRV @@ -118,14 +115,14 @@ libhisi_zip_la_LIBADD = -ldl -lpthread libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread libwd_crypto_la_DEPENDENCIES = libwd.la

-libwd_dae_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread -libwd_dae_la_DEPENDENCIES = libwd.la +#libwd_dae_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread +#libwd_dae_la_DEPENDENCIES = libwd.la

libhisi_sec_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) libhisi_sec_la_DEPENDENCIES = libwd.la libwd_crypto.la

-libhisi_hpre_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) -libhisi_hpre_la_DEPENDENCIES = libwd.la libwd_crypto.la +#libhisi_hpre_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) +#libhisi_hpre_la_DEPENDENCIES = libwd.la libwd_crypto.la

libisa_ce_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) libisa_ce_la_DEPENDENCIES = libwd.la libwd_crypto.la @@ -133,13 +130,13 @@ libisa_ce_la_DEPENDENCIES = libwd.la libwd_crypto.la libisa_sve_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) libisa_sve_la_DEPENDENCIES = libwd.la libwd_crypto.la

-libhisi_dae_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_dae_la_OBJECTS) -libhisi_dae_la_DEPENDENCIES = libwd.la libwd_dae.la +#libhisi_dae_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_dae_la_OBJECTS) +#libhisi_dae_la_DEPENDENCIES = libwd.la libwd_dae.la

else UADK_WD_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd.map UADK_CRYPTO_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_crypto.map -UADK_DAE_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_dae.map +#UADK_DAE_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_dae.map UADK_COMP_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_comp.map UADK_V1_SYMBOL= -Wl,--version-script,$(top_srcdir)/v1/libwd.map

@@ -154,9 +151,9 @@ libwd_crypto_la_LIBADD= -lwd -ldl -lnuma -lm -lpthread libwd_crypto_la_LDFLAGS=$(UADK_VERSION) $(UADK_CRYPTO_SYMBOL) libwd_crypto_la_DEPENDENCIES= libwd.la

-libwd_dae_la_LIBADD= -lwd -ldl -lnuma -lm -lpthread -libwd_dae_la_LDFLAGS=$(UADK_VERSION) $(UADK_DAE_SYMBOL) -libwd_dae_la_DEPENDENCIES= libwd.la +#libwd_dae_la_LIBADD= -lwd -ldl -lnuma -lm -lpthread +#libwd_dae_la_LDFLAGS=$(UADK_VERSION) $(UADK_DAE_SYMBOL) +#libwd_dae_la_DEPENDENCIES= libwd.la

libhisi_zip_la_LIBADD= -lwd -ldl -lwd_comp libhisi_zip_la_LDFLAGS=$(UADK_VERSION) @@ -166,9 +163,9 @@ libhisi_sec_la_LIBADD= -lwd -lwd_crypto libhisi_sec_la_LDFLAGS=$(UADK_VERSION) libhisi_sec_la_DEPENDENCIES= libwd.la libwd_crypto.la

-libhisi_hpre_la_LIBADD= -lwd -lwd_crypto -libhisi_hpre_la_LDFLAGS=$(UADK_VERSION) -libhisi_hpre_la_DEPENDENCIES= libwd.la libwd_crypto.la +#libhisi_hpre_la_LIBADD= -lwd -lwd_crypto +#libhisi_hpre_la_LDFLAGS=$(UADK_VERSION) +#libhisi_hpre_la_DEPENDENCIES= libwd.la libwd_crypto.la

libisa_ce_la_LIBADD= -lwd -lwd_crypto libisa_ce_la_LDFLAGS=$(UADK_VERSION) @@ -178,13 +175,14 @@ libisa_sve_la_LIBADD= -lwd -lwd_crypto libisa_sve_la_LDFLAGS=$(UADK_VERSION) libisa_sve_la_DEPENDENCIES= libwd.la libwd_crypto.la

-libhisi_dae_la_LIBADD= -lwd -lwd_dae -libhisi_dae_la_LDFLAGS=$(UADK_VERSION) -libhisi_dae_la_DEPENDENCIES= libwd.la libwd_dae.la +#libhisi_dae_la_LIBADD= -lwd -lwd_dae +#libhisi_dae_la_LDFLAGS=$(UADK_VERSION) +#libhisi_dae_la_DEPENDENCIES= libwd.la libwd_dae.la

endif # WD_STATIC_DRV

pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = lib/libwd_crypto.pc lib/libwd_comp.pc lib/libwd.pc

-SUBDIRS=. test v1/test uadk_tool sample +#SUBDIRS=. test v1/test uadk_tool sample +SUBDIRS=. uadk_tool \ No newline at end of file diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 8406c699..bb87735d 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -10,6 +10,7 @@ #include "wd_cipher.h" #include "wd_digest.h" #include "wd_aead.h" +#include "isa_ce_sm4.h"

#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 @@ -1567,7 +1568,7 @@ static int fill_digest_long_hash(handle_t h_qp, struct wd_digest_msg *msg, static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_digest_msg *recv_msg) { - struct wd_digest_msg *temp_msg; + struct wd_digest_msg *temp_msg = NULL; __u16 done;

done = sqe->type2.done_flag & SEC_DONE_MASK; @@ -1585,7 +1586,7 @@ static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2002,7 +2003,7 @@ put_sgl: static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, struct wd_digest_msg *recv_msg) { - struct wd_digest_msg *temp_msg; + struct wd_digest_msg *temp_msg = NULL; __u16 done;

done = sqe->done_flag & SEC_DONE_MASK; @@ -2020,7 +2021,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2601,7 +2602,7 @@ static void update_stream_counter(struct wd_aead_msg *recv_msg) static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { - struct wd_aead_msg *temp_msg; + struct wd_aead_msg *temp_msg = NULL; __u16 done, icv;

done = sqe->type2.done_flag & SEC_DONE_MASK; @@ -2622,7 +2623,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2979,7 +2980,7 @@ static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, struct wd_aead_msg *recv_msg) { - struct wd_aead_msg *temp_msg; + struct wd_aead_msg *temp_msg = NULL; __u16 done, icv;

done = sqe->done_flag & SEC_DONE_MASK; @@ -3001,7 +3002,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + //temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -3061,6 +3062,7 @@ static int hisi_sec_init(void *conf, void *priv) return -WD_EINVAL; }

+ WD_ERR("debug: call function: %s!\n", __func__); qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { @@ -3142,6 +3144,7 @@ static void __attribute__((constructor)) hisi_sec2_probe(void) WD_ERR("Error: register SEC %s failed!\n", aead_alg_driver[i].alg_name); } + isa_ce_probe(); }

#ifdef WD_STATIC_DRV @@ -3165,5 +3168,6 @@ static void __attribute__((destructor)) hisi_sec2_remove(void) alg_num = ARRAY_SIZE(aead_alg_driver); for (i = 0; i < alg_num; i++) wd_alg_driver_unregister(&aead_alg_driver[i]); + isa_ce_remove(); }

diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 6475246b..9bbc3d6a 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -11,6 +11,7 @@ * Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */

+#include "wd_alg.h" #include "drv/wd_cipher_drv.h" #include "isa_ce_sm4.h" #include "wd_cipher.h" @@ -428,7 +429,11 @@ static struct wd_alg_driver cipher_alg_driver[] = { GEN_CE_ALG_DRIVER("ecb(sm4)", cipher), };

+#ifdef WD_STATIC_DRV +void isa_ce_probe(void) +#else static void __attribute__((constructor)) isa_ce_probe(void) +#endif { __u32 alg_num, i; int ret; @@ -444,7 +449,11 @@ static void __attribute__((constructor)) isa_ce_probe(void) } }

+#ifdef WD_STATIC_DRV +void isa_ce_remove(void) +#else static void __attribute__((destructor)) isa_ce_remove(void) +#endif { __u32 alg_num, i;

diff --git a/drv/isa_ce_sm4.h b/drv/isa_ce_sm4.h index a80224f5..03dcee17 100644 --- a/drv/isa_ce_sm4.h +++ b/drv/isa_ce_sm4.h @@ -55,6 +55,11 @@ int sm4_v8_xts_decrypt(const unsigned char *in, unsigned char *out, size_t lengt const struct SM4_KEY *key, unsigned char *ivec, const struct SM4_KEY *key2);

+#ifdef WD_STATIC_DRV + void isa_ce_probe(void); + void isa_ce_remove(void); +#endif + #ifdef __cplusplus } #endif diff --git a/include/wd_alg.h b/include/wd_alg.h index f5d1447b..0128506f 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -106,6 +106,7 @@ struct wd_alg_driver { int op_type_num; int priv_size; handle_t fallback; + int init_state;

int (*init)(void *conf, void *priv); void (*exit)(void *priv); diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index d1c00380..90ddc956 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -80,6 +80,7 @@ struct wd_ctx { handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; };

/** @@ -121,6 +122,14 @@ struct wd_ctx_config { struct wd_ctx_nums { __u32 sync_ctx_num; __u32 async_ctx_num; + __u8 ctx_type; +}; + +enum wd_ctx_priority { + UADK_CTX_HW = 0x0, + UADK_CTX_CE_INS = 0x1, + UADK_CTX_SVE_INS = 0x2, + UADK_CTX_SOFT = 0x3 };

/** @@ -150,6 +159,9 @@ struct wd_ctx_internal { __u8 ctx_mode; __u16 sqn; pthread_spinlock_t lock; + __u8 ctx_type; + struct wd_alg_driver *drv; + void *drv_priv; };

struct wd_ctx_config_internal { diff --git a/include/wd_sched.h b/include/wd_sched.h index be541c6d..5a69cb9c 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -32,6 +32,7 @@ struct sched_params { __u8 mode; __u32 begin; __u32 end; + int ctx_prop; };

typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/include/wd_util.h b/include/wd_util.h index bed2189c..29ff3af7 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -125,8 +125,8 @@ struct wd_msg_handle {

struct wd_init_attrs { __u32 sched_type; + __u32 task_type; char *alg; - struct wd_alg_driver *driver; struct wd_sched *sched; struct wd_ctx_params *ctx_params; struct wd_ctx_config *ctx_config; @@ -172,6 +172,9 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in); */ void wd_memset_zero(void *data, __u32 size);

+int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config); +void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config); + /* * wd_init_async_request_pool() - Init async message pools. * @pool: Pointer of message pool. @@ -453,8 +456,8 @@ static inline void wd_alg_clear_init(enum wd_status *status) */ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type); + char *alg, int task_type, enum wd_type type, + int max_op_type);

void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params);

@@ -488,10 +491,8 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv); * * Return 0 if succeed and other error number if fail. */ -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv); -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv); +int wd_alg_init_driver(struct wd_ctx_config_internal *config); +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config);

/** * wd_dlopen_drv() - Open the dynamic library file of the device driver. diff --git a/module.mk b/module.mk index 32fa6f06..96cb4f2d 100644 --- a/module.mk +++ b/module.mk @@ -23,6 +23,7 @@ define uadk_cmd_build $(if $(filter X86, $(HOST_PLATFORM_TYPE)),LDFLAGS="-L$(CI_AARCH64_GNU_LIBS) -lnuma -lm") \ ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes $(abspath $(PRIVATE_PATH))/configure \ --enable-perf=yes \ + --disable-shared --with-static_drv \ --host aarch64-linux-gnu \ --target aarch64-linux-gnu \ --includedir=/usr/local/include/uadk \ @@ -34,32 +35,32 @@ endef

define uadk_cmd_install @mkdir -p $(uadk_get_install_lib_dir) $(uadk_get_install_bin_dir) - cp -pfd $(PRIVATE_BUILD_DIR)/.libs/lib* $(uadk_get_install_lib_dir) - cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto.so.3 $(uadk_get_install_lib_dir) - cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto_wd.so $(uadk_get_install_lib_dir) - cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto_wd_esl.so $(uadk_get_install_lib_dir) - cp -pf $(EXTERNAL_DEVEL)/openssl/libssl.so.3 $(uadk_get_install_lib_dir) - cp -pf $(PRIVATE_BUILD_DIR)/test/hisi_hpre_test/test_hisi_hpre $(uadk_get_install_bin_dir)/test_hpre_sva -# cp -pf $(PRIVATE_BUILD_DIR)/test/hisi_zip_test/zip_sva_perf $(uadk_get_install_bin_dir)/zip_sva_perf - cp -pf $(PRIVATE_BUILD_DIR)/test/wd_mempool_test $(uadk_get_install_bin_dir)/wd_mempool_test - cp -pf $(PRIVATE_BUILD_DIR)/uadk_tool/uadk_tool $(uadk_get_install_bin_dir)/uadk_tool - cp -pf $(PRIVATE_BUILD_DIR)/sample/uadk_sample $(uadk_get_install_bin_dir)/uadk_sample +# cp -pfd $(PRIVATE_BUILD_DIR)/.libs/lib* $(uadk_get_install_lib_dir) +# cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto.so.3 $(uadk_get_install_lib_dir) +# cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto_wd.so $(uadk_get_install_lib_dir) +# cp -pf $(EXTERNAL_DEVEL)/openssl/libcrypto_wd_esl.so $(uadk_get_install_lib_dir) +# cp -pf $(EXTERNAL_DEVEL)/openssl/libssl.so.3 $(uadk_get_install_lib_dir) +# cp -pf $(PRIVATE_BUILD_DIR)/test/hisi_hpre_test/test_hisi_hpre $(uadk_get_install_bin_dir)/test_hpre_sva + cp -pf $(PRIVATE_BUILD_DIR)/test/hisi_zip_test/zip_sva_perf $(uadk_get_install_bin_dir)/zip_sva_perf +# cp -pf $(PRIVATE_BUILD_DIR)/test/wd_mempool_test $(uadk_get_install_bin_dir)/wd_mempool_test +# cp -pf $(PRIVATE_BUILD_DIR)/uadk_tool/uadk_tool $(uadk_get_install_bin_dir)/uadk_tool +# cp -pf $(PRIVATE_BUILD_DIR)/sample/uadk_sample $(uadk_get_install_bin_dir)/uadk_sample

- cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/test_hisi_zip $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/test_hisi_zlib $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/wd_zip_test $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test_sgl/wd_zip_test_sgl $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test_sgl/sgl_test $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_sec_test/test_hisi_sec $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_sec_test_sgl/test_hisi_sec_sgl $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/test_hisi_hpre $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/test_hisi_hpre_times $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/hpre_test_tools $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngu $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngk $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngp $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/test_mm/test_wd_mem $(uadk_get_install_bin_dir) - cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/bmm_test/bmm_test $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/test_hisi_zip $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/test_hisi_zlib $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test/wd_zip_test $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test_sgl/wd_zip_test_sgl $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_zip_test_sgl/sgl_test $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_sec_test/test_hisi_sec $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_sec_test_sgl/test_hisi_sec_sgl $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/test_hisi_hpre $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/test_hisi_hpre_times $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_hpre_test/hpre_test_tools $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngu $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngk $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/hisi_trng_test/test_hisi_trngp $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/test_mm/test_wd_mem $(uadk_get_install_bin_dir) +# cp -rpf $(PRIVATE_BUILD_DIR)/v1/test/bmm_test/bmm_test $(uadk_get_install_bin_dir) endef

LOCAL_DEF_CMD_BUILD := uadk_cmd_build diff --git a/wd.c b/wd.c index 964d3e9f..6da25561 100644 --- a/wd.c +++ b/wd.c @@ -503,7 +503,6 @@ void *wd_ctx_mmap_qfr(handle_t h_ctx, enum uacce_qfrt qfrt) }

size = ctx->qfrs_offs[qfrt]; - addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, ctx->fd, off); if (addr == MAP_FAILED) { WD_ERR("failed to mmap, qfrt = %d, err = %d!\n", qfrt, -errno); diff --git a/wd_cipher.c b/wd_cipher.c index b5a94a77..75dbb216 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -8,6 +8,7 @@ #include <pthread.h> #include <sched.h> #include <limits.h> +#include "wd_alg.h" #include "include/drv/wd_cipher_drv.h" #include "wd_cipher.h"

@@ -52,7 +53,6 @@ struct wd_cipher_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *priv; void *dlhandle; void *dlh_list; @@ -82,20 +82,16 @@ static void wd_cipher_close_driver(int init_type) }

if (wd_cipher_setting.dlhandle) { - wd_release_drv(wd_cipher_setting.driver); dlclose(wd_cipher_setting.dlhandle); wd_cipher_setting.dlhandle = NULL; } #else - wd_release_drv(wd_cipher_setting.driver); hisi_sec2_remove(); #endif }

static int wd_cipher_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "cbc(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -129,15 +125,6 @@ static int wd_cipher_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_cipher_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_cipher_setting.driver = driver; - return WD_SUCCESS; }

@@ -275,7 +262,7 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) }

sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; - ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.config.ctxs[0].drv); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; @@ -326,6 +313,7 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, { int ret;

+ WD_ERR("debug: call function: %s!\n", __func__); ret = wd_set_epoll_en("WD_CIPHER_EPOLL_EN", &wd_cipher_setting.config.epoll_en); if (ret < 0) @@ -346,16 +334,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched;

- ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver, - &wd_cipher_setting.priv); - if (ret) - goto out_clear_pool; - return 0;

-out_clear_pool: - wd_uninit_async_request_pool(&wd_cipher_setting.pool); out_clear_sched: wd_clear_sched(&wd_cipher_setting.sched); out_clear_ctx_config: @@ -376,10 +356,6 @@ static int wd_cipher_common_uninit(void) /* unset config, sched, driver */ wd_clear_sched(&wd_cipher_setting.sched);

- wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver, - &wd_cipher_setting.priv); - return 0; }

@@ -405,10 +381,22 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver;

+ ret = wd_ctx_drv_config("ecb(aes)", &wd_cipher_setting.config); + if (ret) + goto out_uninit_nolock; + + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) + goto out_drv_deconfig; + wd_alg_set_init(&wd_cipher_setting.status);

return 0;

+out_drv_deconfig: + wd_ctx_drv_deconfig(&wd_cipher_setting.config); +out_uninit_nolock: + wd_cipher_common_uninit(); out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: @@ -420,6 +408,9 @@ void wd_cipher_uninit(void) { int ret;

+ wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_drv_deconfig(&wd_cipher_setting.config); + ret = wd_cipher_common_uninit(); if (ret) return; @@ -460,37 +451,27 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p while (ret != 0) { memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal));

- /* Get alg driver and dev name */ - wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_cipher_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ cipher_ctx_params.ctx_set_num = cipher_ctx_num; ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - wd_cipher_setting.driver, + alg, task_type, WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); if (ret) { if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); continue; } - goto out_driver; + goto out_dlclose; }

wd_cipher_init_attrs.alg = alg; wd_cipher_init_attrs.sched_type = sched_type; - wd_cipher_init_attrs.driver = wd_cipher_setting.driver; + wd_cipher_init_attrs.task_type = task_type; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; ret = wd_alg_attrs_init(&wd_cipher_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); wd_ctx_param_uninit(&cipher_ctx_params); continue; } @@ -499,16 +480,28 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p } }

+ WD_ERR("ctxs numbers: %u.\n", wd_cipher_setting.config.ctx_num); + ret = wd_ctx_drv_config(alg, &wd_cipher_setting.config); + if (ret) + goto out_uninit_nolock; + + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) + goto out_drv_deconfig; + wd_alg_set_init(&wd_cipher_setting.status); wd_ctx_param_uninit(&cipher_ctx_params);

return 0;

+out_drv_deconfig: + wd_ctx_drv_deconfig(&wd_cipher_setting.config); +out_uninit_nolock: + wd_cipher_common_uninit(); + wd_alg_attrs_uninit(&wd_cipher_init_attrs); out_params_uninit: wd_ctx_param_uninit(&cipher_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_cipher_setting.driver); -out_dlopen: +out_dlclose: wd_cipher_close_driver(WD_TYPE_V2); out_uninit: wd_alg_clear_init(&wd_cipher_setting.status); @@ -519,12 +512,12 @@ void wd_cipher_uninit2(void) { int ret;

+ wd_ctx_drv_deconfig(&wd_cipher_setting.config); ret = wd_cipher_common_uninit(); if (ret) return;

wd_alg_attrs_uninit(&wd_cipher_init_attrs); - wd_alg_drv_unbind(wd_cipher_setting.driver); wd_cipher_close_driver(WD_TYPE_V2); wd_cipher_setting.dlh_list = NULL; wd_alg_clear_init(&wd_cipher_setting.status); @@ -684,13 +677,13 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_msg_handle msg_handle; int ret;

- msg_handle.send = wd_cipher_setting.driver->send; - msg_handle.recv = wd_cipher_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv;

- wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, UADK_ALG_HW); ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, wd_cipher_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_unlock(ctx, UADK_ALG_HW);

return ret; } @@ -764,7 +757,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id;

- ret = wd_cipher_setting.driver->send(ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -813,7 +806,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_cipher_setting.driver->recv(ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { diff --git a/wd_comp.c b/wd_comp.c index 98f9e7d4..f2b033a3 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -49,7 +49,6 @@ struct wd_comp_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *priv; void *dlhandle; void *dlh_list; @@ -67,20 +66,16 @@ static void wd_comp_close_driver(int init_type) }

if (wd_comp_setting.dlhandle) { - wd_release_drv(wd_comp_setting.driver); dlclose(wd_comp_setting.dlhandle); wd_comp_setting.dlhandle = NULL; } #else - wd_release_drv(wd_comp_setting.driver); hisi_zip_remove(); #endif }

static int wd_comp_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "zlib"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -114,14 +109,6 @@ static int wd_comp_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_comp_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_comp_setting.driver = driver;

return WD_SUCCESS; } @@ -168,16 +155,8 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched;

- ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver, - &wd_comp_setting.priv); - if (ret) - goto out_clear_pool; - return 0;

-out_clear_pool: - wd_uninit_async_request_pool(&wd_comp_setting.pool); out_clear_sched: wd_clear_sched(&wd_comp_setting.sched); out_clear_ctx_config: @@ -198,10 +177,6 @@ static int wd_comp_uninit_nolock(void) /* Unset config, sched, driver */ wd_clear_sched(&wd_comp_setting.sched);

- wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver, - &wd_comp_setting.priv); - return 0; }

@@ -227,10 +202,22 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_driver;

+ ret = wd_ctx_drv_config("zlib", &wd_comp_setting.config); + if (ret) + goto out_uninit_nolock; + + ret = wd_alg_init_driver(&wd_comp_setting.config); + if (ret) + goto out_drv_deconfig; + wd_alg_set_init(&wd_comp_setting.status);

return 0;

+out_drv_deconfig: + wd_ctx_drv_deconfig(&wd_comp_setting.config); +out_uninit_nolock: + wd_comp_uninit_nolock(); out_clear_driver: wd_comp_close_driver(WD_TYPE_V1); out_clear_init: @@ -242,6 +229,9 @@ void wd_comp_uninit(void) { int ret;

+ wd_alg_uninit_driver(&wd_comp_setting.config); + wd_ctx_drv_deconfig(&wd_comp_setting.config); + ret = wd_comp_uninit_nolock(); if (ret) return; @@ -282,36 +272,26 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par while (ret != 0) { memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal));

- /* Get alg driver and dev name */ - wd_comp_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_comp_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlclose; - } - + /* Init ctx param and prepare for ctx request */ comp_ctx_params.ctx_set_num = comp_ctx_num; ret = wd_ctx_param_init(&comp_ctx_params, ctx_params, - wd_comp_setting.driver, WD_COMP_TYPE, WD_DIR_MAX); + alg, task_type, WD_COMP_TYPE, WD_DIR_MAX); if (ret) { if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); continue; } - goto out_unbind_drv; + goto out_dlclose; }

wd_comp_init_attrs.alg = alg; wd_comp_init_attrs.sched_type = sched_type; - wd_comp_init_attrs.driver = wd_comp_setting.driver; + wd_comp_init_attrs.task_type = task_type; wd_comp_init_attrs.ctx_params = &comp_ctx_params; wd_comp_init_attrs.alg_init = wd_comp_init_nolock; wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx; ret = wd_alg_attrs_init(&wd_comp_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); wd_ctx_param_uninit(&comp_ctx_params); continue; } @@ -320,15 +300,26 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par } }

+ ret = wd_ctx_drv_config(alg, &wd_comp_setting.config); + if (ret) + goto out_uninit_nolock; + + ret = wd_alg_init_driver(&wd_comp_setting.config); + if (ret) + goto out_drv_deconfig; + wd_alg_set_init(&wd_comp_setting.status); wd_ctx_param_uninit(&comp_ctx_params);

return 0;

+out_drv_deconfig: + wd_ctx_drv_deconfig(&wd_comp_setting.config); +out_uninit_nolock: + wd_comp_uninit_nolock(); + wd_alg_attrs_uninit(&wd_comp_init_attrs); out_params_uninit: wd_ctx_param_uninit(&comp_ctx_params); -out_unbind_drv: - wd_alg_drv_unbind(wd_comp_setting.driver); out_dlclose: wd_comp_close_driver(WD_TYPE_V2); out_uninit: @@ -345,7 +336,7 @@ void wd_comp_uninit2(void) return;

wd_alg_attrs_uninit(&wd_comp_init_attrs); - wd_alg_drv_unbind(wd_comp_setting.driver); + wd_comp_close_driver(WD_TYPE_V2); wd_comp_setting.dlh_list = NULL; wd_alg_clear_init(&wd_comp_setting.status); @@ -381,7 +372,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_comp_setting.driver->recv(ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -393,7 +384,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg = wd_find_msg_in_pool(&wd_comp_setting.pool, idx, resp_msg.tag); if (unlikely(!msg)) { - WD_ERR("failed to find msg from pool!\n"); + WD_ERR("failed to get msg from pool!\n"); return -WD_EINVAL; }

@@ -611,8 +602,8 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx;

- msg_handle.send = wd_comp_setting.driver->send; - msg_handle.recv = wd_comp_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv;

pthread_spin_lock(&ctx->lock); ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, @@ -872,7 +863,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req)

pthread_spin_lock(&ctx->lock);

- ret = wd_comp_setting.driver->send(ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { pthread_spin_unlock(&ctx->lock); if (ret != -WD_EBUSY) diff --git a/wd_sched.c b/wd_sched.c index 36bf71e3..ddce9f43 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -60,7 +60,10 @@ struct sched_ctx_region { * @valid: the region used flag. */ struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; + struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; // default as HW ctxs + struct sched_ctx_region *ce_ctx_region[SCHED_MODE_BUTT]; + struct sched_ctx_region *sve_ctx_region[SCHED_MODE_BUTT]; + struct sched_ctx_region *soft_ctx_region[SCHED_MODE_BUTT]; bool valid; };

@@ -191,11 +194,13 @@ static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param)

if (!param) { memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; WD_INFO("session don't set scheduler parameters!\n"); } else if (param->numa_id < 0) { skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; + //skey->numa_id = sched_ctx->numa_map[node]; + skey->numa_id = 0; } else { skey->type = param->type; skey->numa_id = param->numa_id; @@ -504,7 +509,7 @@ int wd_sched_rr_instance(const struct wd_sched *sched, { struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; + __u8 type, mode, prop; int numa_id;

if (!sched || !sched->h_sched_ctx || !param) { @@ -540,6 +545,12 @@ int wd_sched_rr_instance(const struct wd_sched *sched, return -WD_EINVAL; }

+ prop = param->ctx_prop; + if (prop > UADK_CTX_SOFT) { + WD_ERR("invalid: sched_ctx's prop is %u\n", prop); + return -WD_EINVAL; + } + sched_info = sched_ctx->sched_info;

if (!sched_info[numa_id].ctx_region[mode]) { @@ -548,16 +559,42 @@ int wd_sched_rr_instance(const struct wd_sched *sched, return -WD_EINVAL; }

- sched_info[numa_id].ctx_region[mode][type].begin = param->begin; - sched_info[numa_id].ctx_region[mode][type].end = param->end; - sched_info[numa_id].ctx_region[mode][type].last = param->begin; - sched_info[numa_id].ctx_region[mode][type].valid = true; - sched_info[numa_id].valid = true; - - wd_sched_map_cpus_to_dev(sched_ctx); - - pthread_mutex_init(&sched_info[numa_id].ctx_region[mode][type].lock, + switch (prop) { + case UADK_CTX_HW: + sched_info[numa_id].ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].ctx_region[mode][type].end = param->end; + sched_info[numa_id].ctx_region[mode][type].last = param->begin; + sched_info[numa_id].ctx_region[mode][type].valid = true; + pthread_mutex_init(&sched_info[numa_id].ctx_region[mode][type].lock, NULL); + break; + case UADK_CTX_CE_INS: + sched_info[numa_id].ce_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].ce_ctx_region[mode][type].end = param->end; + sched_info[numa_id].ce_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].ce_ctx_region[mode][type].valid = true; + pthread_mutex_init(&sched_info[numa_id].ce_ctx_region[mode][type].lock, + NULL); + break; + case UADK_CTX_SVE_INS: + sched_info[numa_id].sve_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].sve_ctx_region[mode][type].end = param->end; + sched_info[numa_id].sve_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].sve_ctx_region[mode][type].valid = true; + pthread_mutex_init(&sched_info[numa_id].sve_ctx_region[mode][type].lock, + NULL); + break; + case UADK_CTX_SOFT: + sched_info[numa_id].soft_ctx_region[mode][type].begin = param->begin; + sched_info[numa_id].soft_ctx_region[mode][type].end = param->end; + sched_info[numa_id].soft_ctx_region[mode][type].last = param->begin; + sched_info[numa_id].soft_ctx_region[mode][type].valid = true; + pthread_mutex_init(&sched_info[numa_id].soft_ctx_region[mode][type].lock, + NULL); + break; + } + wd_sched_map_cpus_to_dev(sched_ctx); + sched_info[numa_id].valid = true;

return 0; } @@ -585,6 +622,21 @@ void wd_sched_rr_release(struct wd_sched *sched) free(sched_info[i].ctx_region[j]); sched_info[i].ctx_region[j] = NULL; } + + if (sched_info[i].ce_ctx_region[j]) { + free(sched_info[i].ce_ctx_region[j]); + sched_info[i].ce_ctx_region[j] = NULL; + } + + if (sched_info[i].sve_ctx_region[j]) { + free(sched_info[i].sve_ctx_region[j]); + sched_info[i].sve_ctx_region[j] = NULL; + } + + if (sched_info[i].soft_ctx_region[j]) { + free(sched_info[i].soft_ctx_region[j]); + sched_info[i].soft_ctx_region[j] = NULL; + } } }

@@ -614,13 +666,74 @@ static int numa_num_check(__u16 numa_num) return 0; }

+static int wd_sched_region_init(struct wd_sched_ctx *sched_ctx, + __u8 type_num, __u16 numa_num) +{ + struct wd_sched_info *sched_info = sched_ctx->sched_info; + int i, j; + int ri, idx; + + for (i = 0; i < numa_num; i++) { + for (j = 0; j < SCHED_MODE_BUTT; j++) { + sched_info[i].ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].ctx_region[j]) + goto hw_err; + + sched_info[i].ce_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].ce_ctx_region[j]) + goto ce_err; + + sched_info[i].sve_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].sve_ctx_region[j]) + goto sve_err; + + sched_info[i].soft_ctx_region[j] = + calloc(1, sizeof(struct sched_ctx_region) * type_num); + if (!sched_info[i].soft_ctx_region[j]) + goto soft_err; + } + } + + return 0; + +soft_err: + free(sched_info[i].sve_ctx_region[j]); + sched_info[i].sve_ctx_region[j] = NULL; +sve_err: + free(sched_info[i].ce_ctx_region[j]); + sched_info[i].ce_ctx_region[j] = NULL; +ce_err: + free(sched_info[i].ctx_region[j]); + sched_info[i].ctx_region[j] = NULL; +hw_err: + for (ri = i - 1; ri >= 0; ri--) { + for (idx = 0; idx < SCHED_MODE_BUTT; idx++) { + free(sched_info[ri].ctx_region[idx]); + sched_info[ri].ctx_region[idx] = NULL; + + free(sched_info[ri].ce_ctx_region[idx]); + sched_info[ri].ce_ctx_region[idx] = NULL; + + free(sched_info[ri].sve_ctx_region[idx]); + sched_info[ri].sve_ctx_region[idx] = NULL; + + free(sched_info[ri].soft_ctx_region[idx]); + sched_info[ri].soft_ctx_region[idx] = NULL; + } + } + + return -WD_EINVAL; +} + struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, __u16 numa_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int i, j; + int ret;

if (numa_num_check(numa_num)) return NULL; @@ -650,15 +763,9 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, sched_type == SCHED_POLICY_SINGLE) goto simple_ok;

- sched_info = sched_ctx->sched_info; - for (i = 0; i < numa_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } - } + ret = wd_sched_region_init(sched_ctx, type_num, numa_num); + if (ret) + goto err_out;

simple_ok: sched_ctx->poll_func = func; diff --git a/wd_util.c b/wd_util.c index 4fcfd34b..64983c41 100644 --- a/wd_util.c +++ b/wd_util.c @@ -167,6 +167,7 @@ static void clone_ctx_to_internal(struct wd_ctx *ctx, ctx_in->ctx = ctx->ctx; ctx_in->op_type = ctx->op_type; ctx_in->ctx_mode = ctx->ctx_mode; + ctx_in->ctx_type = ctx->ctx_type; }

static int wd_shm_create(struct wd_ctx_config_internal *in) @@ -1976,12 +1977,17 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) fb_driver->exit(NULL); }

-int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv) +static int wd_ctx_init_driver(struct wd_ctx_config_internal *config, + struct wd_ctx_internal *ctx_config) { - void *priv; + struct wd_alg_driver *driver = ctx_config->drv; + void *priv = ctx_config->drv_priv; int ret;

+ /* Prevent repeated initialization */ + if (driver->init_state) + return 0; + if (!driver->priv_size) { WD_ERR("invalid: driver priv ctx size is zero!\n"); return -WD_EINVAL; @@ -2004,6 +2010,7 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("driver init failed.\n"); goto err_alloc; } + driver->init_state = 1;

if (driver->fallback) { ret = wd_alg_init_fallback((struct wd_alg_driver *)driver->fallback); @@ -2012,21 +2019,27 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("soft alg driver init failed.\n"); } } - *drv_priv = priv;

return 0;

err_alloc: free(priv); + ctx_config->drv_priv = NULL; return ret; }

-void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv) +static void wd_ctx_uninit_driver(struct wd_ctx_config_internal *config, + struct wd_ctx_internal *ctx_config) { - void *priv = *drv_priv; + struct wd_alg_driver *driver = ctx_config->drv; + void *priv = ctx_config->drv_priv; + + /* Prevent repeated uninitialization */ + if (!driver->init_state) + return;

driver->exit(priv); + driver->init_state = 0; /* Ctx config just need clear once */ wd_clear_ctx_config(config);

@@ -2035,8 +2048,37 @@ void wd_alg_uninit_driver(struct wd_ctx_config_internal *config,

if (priv) { free(priv); - *drv_priv = NULL; + ctx_config->drv_priv = NULL; + } +} + +int wd_alg_init_driver(struct wd_ctx_config_internal *config) +{ + __u32 i, j; + int ret; + + WD_ERR("debug: call function: %s!\n", __func__); + for (i = 0; i < config->ctx_num; i++) { + ret = wd_ctx_init_driver(config, &config->ctxs[i]); + if (ret) + goto init_err; } + + return 0; + +init_err: + for (j = 0; j < i; j++) + wd_ctx_uninit_driver(config, &config->ctxs[j]); + + return ret; +} + +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config) +{ + __u32 i; + + for (i = 0; i < config->ctx_num; i++) + wd_ctx_uninit_driver(config, &config->ctxs[i]); }

void wd_dlclose_drv(void *dlh_list) @@ -2162,12 +2204,12 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params)

int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type) + char *alg, int task_type, enum wd_type type, + int max_op_type) { const char *env_name = wd_env_name[type]; const char *var_s; - int i, ret; + int ret;

ctx_params->bmp = numa_allocate_nodemask(); if (!ctx_params->bmp) { @@ -2177,9 +2219,9 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params,

/* Only hw driver support environment variable */ var_s = secure_getenv(env_name); - if (var_s && strlen(var_s) && driver->calc_type == UADK_ALG_HW) { + if (var_s && strlen(var_s) && task_type <= TASK_HW) { /* environment variable has the highest priority */ - ret = wd_env_set_ctx_nums(driver->alg_name, env_name, var_s, + ret = wd_env_set_ctx_nums(alg, env_name, var_s, ctx_params, max_op_type); if (ret) { WD_ERR("fail to init ctx nums from %s!\n", env_name); @@ -2202,19 +2244,21 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, return 0; }

+#if 0 /* user_ctx_params is also not set, use driver's defalut queue_num */ numa_bitmask_setall(ctx_params->bmp); for (i = 0; i < driver->op_type_num; i++) { ctx_params->ctx_set_num[i].sync_ctx_num = driver->queue_num; ctx_params->ctx_set_num[i].async_ctx_num = driver->queue_num; } - }

- ctx_params->op_type_num = driver->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { - WD_ERR("fail to check driver op type numbers.\n"); - numa_free_nodemask(ctx_params->bmp); - return -WD_EAGAIN; + ctx_params->op_type_num = driver->op_type_num; + if (ctx_params->op_type_num > (__u32)max_op_type) { + WD_ERR("fail to check driver op type numbers.\n"); + numa_free_nodemask(ctx_params->bmp); + return -WD_EAGAIN; + } +#endif }

return 0; @@ -2353,7 +2397,7 @@ struct wd_alg_driver *wd_alg_drv_bind(int task_type, char *alg_name) case TASK_INSTR: drv = wd_request_drv(alg_name, true); if (!drv) { - WD_ERR("no soft %s driver support\n", alg_name); + WD_ERR("no instr soft %s driver support\n", alg_name); return NULL; } set_driver = drv; @@ -2413,6 +2457,40 @@ static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) return count; }

+void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config) +{ + __u32 i; + + // wd_dlclose_drv after this + + for (i = 0; i < ctx_config->ctx_num; i++) { + wd_alg_drv_unbind(ctx_config->ctxs[i].drv); + } +} + +int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config) +{ + __u32 i, j; + + // wd_dlopen_drv before this + WD_ERR("debug: call function: %s!\n", __func__); + for (i = 0; i < ctx_config->ctx_num; i++) { + ctx_config->ctxs[i].drv = wd_alg_drv_bind(ctx_config->ctxs[i].ctx_type, alg_name); + if (!ctx_config->ctxs[i].drv) { + WD_ERR("failed to bind %s driver.\n", alg_name); + goto bind_err; + } + } + + return 0; + +bind_err: + for (j = 0; j < i; j++) { + wd_alg_drv_unbind(ctx_config->ctxs[j].drv); + } + return -WD_EINVAL; +} + struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) { struct uacce_dev_list *p, *node, *result = NULL; @@ -2507,6 +2585,7 @@ static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *l ctx_config->ctxs[i].ctx_mode = ((i - idx) < ctx_nums.sync_ctx_num) ? CTX_MODE_SYNC : CTX_MODE_ASYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_HW; }

return 0; @@ -2534,6 +2613,7 @@ static int wd_instance_sched_set(struct wd_sched *sched, struct wd_ctx_nums ctx_ sparams.type = op_type; sparams.mode = i; sparams.begin = idx + ctx_nums.sync_ctx_num * i; + sparams.ctx_prop = UADK_CTX_HW; end = idx - 1 + ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num * i; if (end < 0 || sparams.begin > (__u32)end) continue; @@ -2657,17 +2737,51 @@ out_freelist: return ret; }

-static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +static int wd_alg_ce_ctx_init(struct wd_ctx_config *ctx_config, bool fb_flag) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; + __u32 old_num = 0; + size_t buf_size;

- ctx_config->ctx_num = 1; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { - return -WD_ENOMEM; - WD_ERR("failed to alloc ctxs!\n"); + if (!fb_flag) { + ctx_config->ctx_num = 1; + buf_size = sizeof(struct wd_ctx); + ctx_config->ctxs = calloc(ctx_config->ctx_num, buf_size); + if (!ctx_config->ctxs) { + WD_ERR("failed to alloc ctxs!\n"); + return -WD_ENOMEM; + } + + ctx_config->ctxs[0].op_type = 0; + ctx_config->ctxs[0].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[0].ctx_type = UADK_CTX_CE_INS; + ctx_config->ctxs[0].ctx = (handle_t)calloc(1, sizeof(struct wd_ctx)); + if (!ctx_config->ctxs[0].ctx) { + free(ctx_config->ctxs); + WD_ERR("failed to alloc ctx!\n"); + return -WD_ENOMEM; + } + } else { + old_num = ctx_config->ctx_num; + ctx_config->ctx_num += 1; + + buf_size = (old_num + 1) * sizeof(struct wd_ctx); + ctx_config->ctxs = realloc(ctx_config->ctxs, buf_size); + if (!ctx_config->ctxs) { + WD_ERR("failed to realloc ce ctxs!\n"); + return -WD_ENOMEM; + } + + ctx_config->ctxs[buf_size - 1].op_type = 0; + ctx_config->ctxs[buf_size - 1].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[buf_size - 1].ctx_type = UADK_CTX_CE_INS; + ctx_config->ctxs[buf_size - 1].ctx = (handle_t)calloc(1, sizeof(struct wd_ctx)); + if (!ctx_config->ctxs[buf_size - 1].ctx) { + free(ctx_config->ctxs); + ctx_config->ctxs = NULL; + WD_ERR("failed to alloc ctx!\n"); + return -WD_ENOMEM; + } } - ctx_config->ctxs[0].ctx = (handle_t)calloc(1, sizeof(struct wd_ce_ctx));

return WD_SUCCESS; } @@ -2678,7 +2792,7 @@ static void wd_alg_ce_ctx_uninit(struct wd_ctx_config *ctx_config)

for (i = 0; i < ctx_config->ctx_num; i++) { if (ctx_config->ctxs[i].ctx) { - free((struct wd_ce_ctx *)ctx_config->ctxs[i].ctx); + free((struct wd_ctx *)ctx_config->ctxs[i].ctx); ctx_config->ctxs[i].ctx = 0; } } @@ -2700,30 +2814,64 @@ static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) free(ctx_config->ctxs); }

-static int wd_alg_init_sve_ctx(struct wd_ctx_config *ctx_config) +static int wd_alg_init_sve_ctx(struct wd_ctx_config *ctx_config, bool fb_flag) { - struct wd_soft_ctx *ctx_sync, *ctx_async; + struct wd_ctx *ctx_sync, *ctx_async; + __u32 old_num = 0; + size_t buf_size;

- ctx_config->ctx_num = WD_SOFT_CTX_NUM; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) - return -WD_ENOMEM; + if (!fb_flag) { + ctx_config->ctx_num = WD_SOFT_CTX_NUM; + ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); + if (!ctx_config->ctxs) + return -WD_ENOMEM;

- ctx_sync = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_sync) - goto free_ctxs; + ctx_sync = calloc(1, sizeof(struct wd_ctx)); + if (!ctx_sync) + goto free_ctxs;

- ctx_config->ctxs[WD_SOFT_SYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + ctx_config->ctxs[WD_SOFT_SYNC_CTX].op_type = 0; + ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_type = UADK_CTX_SVE_INS;

- ctx_async = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_async) - goto free_ctx_sync; + ctx_async = calloc(1, sizeof(struct wd_ctx)); + if (!ctx_async) + goto free_ctx_sync;

- ctx_config->ctxs[WD_SOFT_ASYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; + ctx_config->ctxs[WD_SOFT_ASYNC_CTX].op_type = 0; + ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; + ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; + ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_type = UADK_CTX_SVE_INS; + } else { + old_num = ctx_config->ctx_num; + ctx_config->ctx_num += WD_SOFT_CTX_NUM; + + buf_size = (old_num + WD_SOFT_CTX_NUM) * sizeof(struct wd_ctx); + ctx_config->ctxs = realloc(ctx_config->ctxs, buf_size); + if (!ctx_config->ctxs) { + WD_ERR("failed to realloc sve ctxs!\n"); + return -WD_ENOMEM; + } + + ctx_sync = calloc(1, sizeof(struct wd_ctx)); + if (!ctx_sync) + goto free_ctxs; + + ctx_config->ctxs[buf_size - WD_SOFT_SYNC_CTX].op_type = 0; + ctx_config->ctxs[buf_size - WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[buf_size - WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + ctx_config->ctxs[buf_size - WD_SOFT_SYNC_CTX].ctx_type = UADK_CTX_SVE_INS; + + ctx_async = calloc(1, sizeof(struct wd_ctx)); + if (!ctx_async) + goto free_ctx_sync; + + ctx_config->ctxs[buf_size - WD_SOFT_ASYNC_CTX].op_type = 0; + ctx_config->ctxs[buf_size - WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; + ctx_config->ctxs[buf_size - WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; + ctx_config->ctxs[buf_size - WD_SOFT_ASYNC_CTX].ctx_type = UADK_CTX_SVE_INS; + }

return 0;

@@ -2734,11 +2882,33 @@ free_ctxs: return -WD_ENOMEM; }

-static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) +static __u32 wd_ctx_get_param(struct wd_ctx_params *ctx_params, + struct wd_alg_driver *driver) { - free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); - free(ctx_config->ctxs); + struct wd_alg_driver *fb_drv = (struct wd_alg_driver *)driver->fallback; + __u32 op_type_num, queue_num; + __u32 i; + + if (ctx_params->op_type_num) + return ctx_params->op_type_num; + + /* User_ctx_params is also not set, use driver's defalut queue_num */ + if (fb_drv && fb_drv->op_type_num > driver->op_type_num) { + op_type_num = fb_drv->op_type_num; + queue_num = fb_drv->queue_num; + } else { + op_type_num = driver->op_type_num; + queue_num = driver->queue_num; + } + + numa_bitmask_setall(ctx_params->bmp); + for (i = 0; i < op_type_num; i++) { + ctx_params->ctx_set_num[i].sync_ctx_num = queue_num; + ctx_params->ctx_set_num[i].async_ctx_num = queue_num; + } + ctx_params->op_type_num = op_type_num; + + return op_type_num; }

int wd_alg_attrs_init(struct wd_init_attrs *attrs) @@ -2748,116 +2918,188 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) __u32 sched_type = attrs->sched_type; struct wd_ctx_config *ctx_config = NULL; struct wd_sched *alg_sched = NULL; + struct wd_alg_driver *drv_fb = NULL; + struct wd_alg_driver *drv = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; - int driver_type = UADK_ALG_HW; - char *alg = attrs->alg; + char *alg_name = attrs->alg; + int task_type = attrs->task_type; + __u32 op_type_num; + struct sched_params sparams; int ret = 0;

if (!attrs->ctx_params) return -WD_EINVAL;

- if (attrs->driver) - driver_type = attrs->driver->calc_type; + ctx_config = calloc(1, sizeof(*ctx_config)); + if (!ctx_config) { + WD_ERR("fail to alloc ctx config\n"); + return -WD_ENOMEM; + } + attrs->ctx_config = ctx_config;

- switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; + /* Get alg driver and dev name */ + switch (task_type) { + case TASK_INSTR: + drv = wd_request_drv(alg_name, true); + if (!drv) { + WD_ERR("no instr soft %s driver support\n", alg_name); + goto out_ctx_config; } - attrs->ctx_config = ctx_config;

- /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); + drv->fallback = 0; + break; + case TASK_HW: + case TASK_MIX: + drv = wd_request_drv(alg_name, false); + if (!drv) { + WD_ERR("no HW %s driver support\n", alg_name); goto out_ctx_config; }

- attrs->sched = alg_sched; - - ret = wd_alg_ce_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ce ctx\n"); - goto out_freesched; + drv->fallback = 0; + if (task_type == TASK_MIX) { + drv_fb = wd_request_drv(alg_name, true); + if (!drv_fb) { + drv->fallback = 0; + WD_ERR("no soft %s driver support\n", alg_name); + } else { + drv->fallback = (handle_t)drv_fb; + WD_ERR("successful to get soft driver\n"); + } } + break; + default: + WD_ERR("task type error.\n"); + goto out_ctx_config; + }

- ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; + /* Get op_type_num */ + op_type_num = wd_ctx_get_param(attrs->ctx_params, drv); + if (!op_type_num) + goto out_ctx_config;

- break; - case UADK_ALG_SVE_INSTR: - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_SINGLE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - return -WD_EINVAL; - } - attrs->sched = alg_sched; + /* Use default sched_type to alloc scheduler */ + alg_sched = wd_sched_rr_alloc(sched_type, op_type_num, + numa_max_node() + 1, alg_poll_func); + if (!alg_sched) { + WD_ERR("fail to alloc scheduler\n"); + goto out_ctx_config; + } + attrs->sched = alg_sched;

- ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - goto out_freesched; - } - attrs->ctx_config = ctx_config; + /* CTX type corresponds to Driver type */ + switch (drv->calc_type) { + case UADK_ALG_HW: + wd_get_alg_type(alg_name, alg_type); + attrs->alg = alg_type;

- ret = wd_alg_init_sve_ctx(ctx_config); + ret = wd_alg_ctx_init(attrs); if (ret) { - WD_ERR("fail to init sve ctx!\n"); + WD_ERR("fail to init ctx\n"); goto out_freesched; }

- ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); + break; + case UADK_ALG_SOFT: + case UADK_ALG_CE_INSTR: + ret = wd_alg_ce_ctx_init(ctx_config, false); if (ret) { - wd_alg_uninit_sve_ctx(ctx_config); + WD_ERR("fail to init ce ctx\n"); goto out_freesched; } - break; - case UADK_ALG_HW: - wd_get_alg_type(alg, alg_type); - attrs->alg = alg_type;

- ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; - - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - numa_max_node() + 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to instance scheduler\n"); - ret = -WD_EINVAL; - goto out_ctx_config; + sparams.begin = 0; + sparams.end = 0; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_pre_init; } - attrs->sched = alg_sched;

- ret = wd_alg_ctx_init(attrs); + break; + case UADK_ALG_SVE_INSTR: + ret = wd_alg_init_sve_ctx(ctx_config, false); if (ret) { - WD_ERR("fail to init ctx\n"); + WD_ERR("fail to init sve ctx!\n"); goto out_freesched; }

- ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) + sparams.begin = 0; + sparams.end = WD_SOFT_CTX_NUM - 1; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); goto out_pre_init; + } + break; default: - WD_ERR("driver type error: %d\n", driver_type); + WD_ERR("driver type error: %d\n", drv->calc_type); return -WD_EINVAL; }

+ ctx_config->cap = attrs->ctx_params->cap; + ret = alg_init_func(ctx_config, alg_sched); + if (ret) + goto out_pre_init; + + /* If there are two type of queue */ + if (drv->fallback) { + /* Fallback driver has no HW driver type */ + switch (drv_fb->calc_type) { + case UADK_ALG_SOFT: + case UADK_ALG_CE_INSTR: + ret = wd_alg_ce_ctx_init(ctx_config, true); + if (ret) { + WD_ERR("fail to init ce ctx\n"); + goto out_freesched; + } + + sparams.begin = 0; + sparams.end = 0; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_pre_init; + } + + break; + case UADK_ALG_SVE_INSTR: + ret = wd_alg_init_sve_ctx(ctx_config, true); + if (ret) { + WD_ERR("fail to init sve ctx!\n"); + goto out_freesched; + } + + sparams.begin = 0; + sparams.end = WD_SOFT_CTX_NUM - 1; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_pre_init; + } + + break; + default: + WD_ERR("driver type error: %d\n", drv->calc_type); + return -WD_EINVAL; + } + + } + WD_ERR("---->ctx nums: %u\n", ctx_config->ctx_num); + return 0;

out_pre_init: - if (driver_type == UADK_ALG_CE_INSTR || driver_type == UADK_ALG_SOFT) + if (drv->calc_type == UADK_ALG_CE_INSTR || drv->calc_type == UADK_ALG_SOFT) wd_alg_ce_ctx_uninit(ctx_config); else wd_alg_ctx_uninit(ctx_config); @@ -2869,31 +3111,44 @@ out_ctx_config: return ret; }

+static void wd_alg_ctxs_uninit(struct wd_ctx_config *ctx_config) +{ + __u32 i; + + for (i = 0; i < ctx_config->ctx_num; i++) { + if (!ctx_config->ctxs[i].ctx) + continue; + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_HW) { + wd_release_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_CE_INS) { + free((struct wd_ctx *)ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_SVE_INS) { + free((struct wd_ctx *)ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + } + + free(ctx_config->ctxs); +} + void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) { struct wd_ctx_config *ctx_config = attrs->ctx_config; struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type;

if (!ctx_config) { wd_sched_rr_release(alg_sched); return; }

- switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - wd_alg_ce_ctx_uninit(ctx_config); - break; - case UADK_ALG_SVE_INSTR: - wd_alg_uninit_sve_ctx(ctx_config); - break; - case UADK_ALG_HW: - wd_alg_ctx_uninit(ctx_config); - break; - default: - break; - } + wd_alg_ctxs_uninit(ctx_config);

free(ctx_config); wd_sched_rr_release(alg_sched);

Adjust scheduler queue processing after unified scheduling

Signed-off-by: Longfang Liu liulongfang@huawei.com --- include/wd_alg_common.h | 1 + wd_cipher.c | 51 ++++++++++++++++++++++++++++++----------- wd_sched.c | 48 ++++++++++++++++++++++++++++++++++---- wd_util.c | 2 ++ 4 files changed, 85 insertions(+), 17 deletions(-)

diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index 90ddc956..75b3bfd9 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -160,6 +160,7 @@ struct wd_ctx_internal { __u16 sqn; pthread_spinlock_t lock; __u8 ctx_type; + __u32 req_nums; struct wd_alg_driver *drv; void *drv_priv; }; diff --git a/wd_cipher.c b/wd_cipher.c index 75dbb216..282604a1 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -729,6 +729,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; struct wd_ctx_internal *ctx; struct wd_cipher_msg *msg; + static int soft_slice = 0; int msg_id, ret; __u32 idx;

@@ -747,22 +748,41 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req)

ctx = config->ctxs + idx;

- msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, - (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("busy, failed to get msg from pool!\n"); - return -WD_EBUSY; - } +try_again: + if (soft_slice > 0) { + soft_slice--; + struct wd_alg_driver *fb_drv; + struct wd_cipher_msg msg; + //WD_ERR("busy, failed to get msg from pool!\n"); + + fill_request_msg(&msg, req, sess); + fb_drv = (struct wd_alg_driver *)ctx->drv->fallback; + ret = fb_drv->send(ctx->ctx, &msg); + if (unlikely(ret < 0)) { + WD_ERR("wd cipher soft async send err!\n"); + return ret; + }

- fill_request_msg(msg, req, sess); - msg->tag = msg_id; + req->cb(req, req->cb_param); + __atomic_add_fetch(&ctx->req_nums, 0x1, __ATOMIC_RELAXED); + } else { + msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, + (void **)&msg); + if (unlikely(msg_id > 0)) { + fill_request_msg(msg, req, sess); + msg->tag = msg_id;

- ret = ctx->drv->send(ctx->ctx, msg); - if (unlikely(ret < 0)) { - if (ret != -WD_EBUSY) - WD_ERR("wd cipher async send err!\n"); + ret = ctx->drv->send(ctx->ctx, msg); + if (unlikely(ret < 0)) { + if (ret != -WD_EBUSY) + WD_ERR("wd cipher async send err!\n");

- goto fail_with_msg; + goto fail_with_msg; + } + } else { + soft_slice = 8; + goto try_again; + } }

wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); @@ -832,6 +852,11 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) *count = recv_count; } while (--tmp);

+ /* Add soft fallback completed request */ + recv_count = ctx->req_nums; + __atomic_sub_fetch(&ctx->req_nums, recv_count, __ATOMIC_RELAXED); + *count += recv_count; + return ret; }

diff --git a/wd_sched.c b/wd_sched.c index ddce9f43..f0deb5f2 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -33,6 +33,7 @@ struct sched_key { __u8 mode; __u32 sync_ctxid; __u32 async_ctxid; + __u32 fb_ctxid; };

/* @@ -168,6 +169,31 @@ static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched return sched_get_next_pos_rr(region, NULL); }

+static __u32 session_sched_fb_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, + const int sched_mode) +{ + struct sched_ctx_region *region = NULL; + struct wd_sched_info *sched_info; + int numa_id; + + sched_info = sched_ctx->sched_info; + if (key->numa_id >= 0 && + sched_info[key->numa_id].ce_ctx_region[key->mode][key->type].valid) { + region = &sched_info[key->numa_id].ce_ctx_region[key->mode][key->type]; + return sched_get_next_pos_rr(region, NULL); + } + + /* If the key->numa_id is not exist, we should scan for a region */ + for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { + if (sched_info[numa_id].ce_ctx_region[key->mode][key->type].valid) { + region = &sched_info[numa_id].ce_ctx_region[key->mode][key->type]; + return sched_get_next_pos_rr(region, NULL); + } + } + + return INVALID_POS; +} + static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; @@ -211,12 +237,19 @@ static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) goto out; }

+ if (param->ctx_prop > 0) { // not HW task + // Todo Perform ctxid initialization and acquisition processing + // based on ctx property + ; + } + skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); goto out; } + skey->fb_ctxid = session_sched_fb_ctx(sched_ctx, skey, CTX_MODE_SYNC);

return (handle_t)skey;

@@ -237,16 +270,23 @@ static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, const int sched_mode) { struct sched_key *key = (struct sched_key *)sched_key; + static int soft_slice = 0;

if (unlikely(!h_sched_ctx || !key)) { WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; }

- /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + if (soft_slice == 0) { + soft_slice = 1; + /* return in do task */ + if (sched_mode == CTX_MODE_SYNC) + return key->sync_ctxid; + return key->async_ctxid; + } else { + soft_slice = 0; + return key->fb_ctxid; + } }

static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, diff --git a/wd_util.c b/wd_util.c index 64983c41..5cbc6571 100644 --- a/wd_util.c +++ b/wd_util.c @@ -1962,6 +1962,7 @@ static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) return -WD_EINVAL; }

+ WD_ERR("debug: call function: %s!\n", __func__); fb_driver->init(NULL, NULL);

return 0; @@ -2742,6 +2743,7 @@ static int wd_alg_ce_ctx_init(struct wd_ctx_config *ctx_config, bool fb_flag) __u32 old_num = 0; size_t buf_size;

+ WD_ERR("debug: call function: %s!\n", __func__); if (!fb_flag) { ctx_config->ctx_num = 1; buf_size = sizeof(struct wd_ctx);