According to the logical layering of UADK, the device driver has been updated from the previous fixed binding HiSilicon accelerator to the dynamic registration method through the algorithm linked list method.

After the update, it can support the use of instruction acceleration and third-party device drivers.

Signed-off-by: Longfang Liu liulongfang@huawei.com --- Makefile.am | 4 +- include/wd_alg.h | 95 ++++++++++++++ include/wd_alg_common.h | 1 + libwd.map | 8 ++ wd_alg.c | 265 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 include/wd_alg.h create mode 100644 wd_alg.c

diff --git a/Makefile.am b/Makefile.am index 4ef5c2c..5b45626 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,7 +33,7 @@ AM_CFLAGS+= -DUADK_RELEASED_TIME=""Released ${MONTH} ${DAY}, ${YEAR}"" pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \ include/wd_comp.h include/wd_dh.h include/wd_digest.h \ include/wd_rsa.h include/uacce.h include/wd_alg_common.h \ - include/wd_ecc.h include/wd_sched.h + include/wd_ecc.h include/wd_sched.h include/wd_alg.h

nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \ v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h @@ -41,7 +41,7 @@ nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libhisi_zip.la \ libhisi_hpre.la libhisi_sec.la

-libwd_la_SOURCES=wd.c wd_mempool.c wd.h \ +libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ v1/wd_rng.c v1/wd_rng.h \ v1/wd_rsa.c v1/wd_rsa.h \ diff --git a/include/wd_alg.h b/include/wd_alg.h new file mode 100644 index 0000000..e369021 --- /dev/null +++ b/include/wd_alg.h @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright 2022 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#ifndef __WD_ALG_H +#define __WD_ALG_H +#include <errno.h> +#include <fcntl.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <syslog.h> +#include <unistd.h> +#include <asm/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define handle_t uintptr_t +enum alg_priority { + UADK_ALG_SOFT = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_HW = 0x3 +}; + +/* + * @drv_name: name of the current device driver + * @alg_name: name of the algorithm supported by the driver + * @priority: priority of the type of algorithm supported by the driver + * @queue_num: number of device queues required by the device to + * execute the algorithm business + * @op_type_num: number of modes in which the device executes the + * algorithm business and requires queues to be executed separately + * @priv_size: parameter memory size passed between the internal + * interfaces of the driver + * @fallback: soft calculation driver handle when performing soft + * calculation supplement + * @init: callback interface for initializing device drivers + * @exit: callback interface for destroying device drivers + * @send: callback interface used to send task packets to + * hardware devices. + * @recv: callback interface used to retrieve the calculation + * result of the task packets from the hardware device. + * @get_usage: callback interface used to obtain the + * utilization rate of hardware devices. + */ +struct wd_alg_driver { + const char *drv_name; + const char *alg_name; + int priority; + int queue_num; + int op_type_num; + int priv_size; + handle_t fallback; + + int (*init)(void *conf, void *priv); + void (*exit)(void *priv); + int (*send)(handle_t ctx, void *drv_msg); + int (*recv)(handle_t ctx, void *drv_msg); + int (*get_usage)(void *param); +}; + +int wd_alg_driver_register(struct wd_alg_driver *drv); +void wd_alg_driver_unregister(struct wd_alg_driver *drv); + +struct wd_alg_list { + const char *alg_name; + const char *drv_name; + int priority; + bool available; + int refcnt; + + struct wd_alg_driver *drv; + struct wd_alg_list *next; +}; + +struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask); +void wd_release_drv(struct wd_alg_driver *drv); + +bool wd_drv_alg_support(const char *alg_name, + struct wd_alg_driver *drv); +void wd_enable_drv(struct wd_alg_driver *drv); +void wd_disable_drv(struct wd_alg_driver *drv); + +struct wd_alg_list *wd_get_alg_head(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index 56539cc..d04b046 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -10,6 +10,7 @@ #include <pthread.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h"

#ifdef __cplusplus extern "C" { diff --git a/libwd.map b/libwd.map index 459f9ba..5522ec0 100644 --- a/libwd.map +++ b/libwd.map @@ -41,5 +41,13 @@ global: wd_add_dev_to_list; wd_find_dev_by_numa;

+ wd_alg_driver_register; + wd_alg_driver_unregister; + wd_request_drv; + wd_release_drv; + wd_drv_alg_support; + wd_enable_drv; + wd_disable_drv; + wd_get_alg_head; local: *; }; diff --git a/wd_alg.c b/wd_alg.c new file mode 100644 index 0000000..197af6f --- /dev/null +++ b/wd_alg.c @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2022 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#define _GNU_SOURCE +#include <ctype.h> +#include <dirent.h> +#include <errno.h> +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> + +#include "wd.h" +#include "wd_alg.h" + +#define SYS_CLASS_DIR "/sys/class/uacce" +static struct wd_alg_list alg_list_head; +static struct wd_alg_list *alg_list_tail = &alg_list_head; + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + +static bool wd_check_accel_dev(const char *drv_name) +{ + struct dirent *dev_dir; + DIR *wd_class; + + wd_class = opendir(SYS_CLASS_DIR); + if (!wd_class) { + WD_ERR("UADK framework isn't enabled in system!\n"); + return NULL; + } + + while ((dev_dir = readdir(wd_class)) != NULL) { + if (!strncmp(dev_dir->d_name, ".", LINUX_CRTDIR_SIZE) || + !strncmp(dev_dir->d_name, "..", LINUX_PRTDIR_SIZE)) + continue; + + if (!strncmp(dev_dir->d_name, drv_name, strlen(drv_name))) { + closedir(wd_class); + return true; + } + } + closedir(wd_class); + + return false; +} + +int wd_alg_driver_register(struct wd_alg_driver *drv) +{ + struct wd_alg_list *new_alg; + + if (!drv) { + WD_ERR("invalid: register drv is NULL!\n"); + return -WD_EINVAL; + } + + new_alg = calloc(1, sizeof(struct wd_alg_list)); + if (!new_alg) { + WD_ERR("failed to alloc alg driver memory!\n"); + return -WD_ENOMEM; + } + + new_alg->alg_name = drv->alg_name; + new_alg->drv_name = drv->drv_name; + new_alg->priority = drv->priority; + new_alg->drv = drv; + new_alg->refcnt = 0; + new_alg->next = NULL; + + if (drv->priority == UADK_ALG_HW) { + /* If not find dev, remove this driver node */ + new_alg->available = wd_check_accel_dev(drv->drv_name); + if (!new_alg->available) { + free(new_alg); + WD_ERR("failed to find alg driver's device!\n"); + return -WD_ENODEV; + } + } else { + /* Should find the CPU if not support SVE or CE */ + new_alg->available = true; + } + + pthread_mutex_lock(&mutex); + alg_list_tail->next = new_alg; + alg_list_tail = new_alg; + pthread_mutex_unlock(&mutex); + + return 0; +} + +void wd_alg_driver_unregister(struct wd_alg_driver *drv) +{ + struct wd_alg_list *npre = &alg_list_head; + struct wd_alg_list *pnext = npre->next; + + /* Alg driver list has no drivers */ + if (!pnext || !drv) + return; + + pthread_mutex_lock(&mutex); + while (pnext) { + if (!strcmp(drv->alg_name, pnext->alg_name) && + !strcmp(drv->drv_name, pnext->drv_name) && + drv->priority == pnext->priority) { + break; + } + npre = pnext; + pnext = pnext->next; + } + + /* The current algorithm is not registered */ + if (!pnext) { + pthread_mutex_unlock(&mutex); + return; + } + + /* Used to locate the problem and ensure symmetrical use driver */ + if (pnext->refcnt > 0) + WD_ERR("driver<%s> still in used: %d\n", pnext->drv_name, pnext->refcnt); + + if (pnext == alg_list_tail) + alg_list_tail = npre; + + npre->next = pnext->next; + free(pnext); + pthread_mutex_unlock(&mutex); +} + +struct wd_alg_list *wd_get_alg_head(void) +{ + return &alg_list_head; +} + +bool wd_drv_alg_support(const char *alg_name, + struct wd_alg_driver *drv) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + + while (pnext) { + if (!strcmp(alg_name, pnext->alg_name) && + !strcmp(drv->drv_name, pnext->drv_name)) { + return true; + } + pnext = pnext->next; + } + + return false; +} + +void wd_enable_drv(struct wd_alg_driver *drv) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + + if (!pnext || !drv) + return; + + pthread_mutex_lock(&mutex); + while (pnext) { + if (!strcmp(drv->alg_name, pnext->alg_name) && + !strcmp(drv->drv_name, pnext->drv_name) && + drv->priority == pnext->priority) { + break; + } + pnext = pnext->next; + } + + if (drv->priority == UADK_ALG_HW) { + /* If not find dev, remove this driver node */ + pnext->available = wd_check_accel_dev(drv->drv_name); + } else { + /* Should find the CPU if not support SVE or CE */ + pnext->available = true; + } + pthread_mutex_unlock(&mutex); +} + +void wd_disable_drv(struct wd_alg_driver *drv) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + + if (!pnext || !drv) + return; + + pthread_mutex_lock(&mutex); + while (pnext) { + if (!strcmp(drv->alg_name, pnext->alg_name) && + !strcmp(drv->drv_name, pnext->drv_name) && + drv->priority == pnext->priority) { + break; + } + pnext = pnext->next; + } + + pnext->available = false; + pthread_mutex_unlock(&mutex); +} + +struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + struct wd_alg_list *select_node = NULL; + struct wd_alg_driver *drv = NULL; + int tmp_priority = -1; + + if (!pnext || !alg_name) { + WD_ERR("invalid: request alg param is error!\n"); + return NULL; + } + + /* Check the list to get an best driver */ + pthread_mutex_lock(&mutex); + while (pnext) { + /* hw_mask true mean not to used hardware dev */ + if (hw_mask && pnext->drv->priority == UADK_ALG_HW) { + pnext = pnext->next; + continue; + } + + if (!strcmp(alg_name, pnext->alg_name) && pnext->available && + pnext->drv->priority > tmp_priority) { + tmp_priority = pnext->drv->priority; + select_node = pnext; + drv = pnext->drv; + } + pnext = pnext->next; + } + + if (select_node) + select_node->refcnt++; + pthread_mutex_unlock(&mutex); + + return drv; +} + +void wd_release_drv(struct wd_alg_driver *drv) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + struct wd_alg_list *select_node = NULL; + + if (!pnext || !drv) + return; + + pthread_mutex_lock(&mutex); + while (pnext) { + if (!strcmp(drv->alg_name, pnext->alg_name) && + !strcmp(drv->drv_name, pnext->drv_name) && + drv->priority == pnext->priority) { + select_node = pnext; + break; + } + pnext = pnext->next; + } + + if (select_node && select_node->refcnt > 0) + select_node->refcnt--; + pthread_mutex_unlock(&mutex); +} +

After the dynamic loading function is added, device resource initialization, driver acquisition, and scheduler initialization functions need to be extracted into the public framework.

so that the algorithm can quickly adapt to the dynamic loading function internally.

Signed-off-by: Longfang Liu liulongfang@huawei.com --- include/wd_alg_common.h | 10 + include/wd_sched.h | 6 +- include/wd_util.h | 63 ++++- wd_comp.c | 2 +- wd_sched.c | 105 +++++++- wd_util.c | 547 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 723 insertions(+), 10 deletions(-)

diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index d04b046..31208ad 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -28,6 +28,13 @@ extern "C" { #define CTX_TYPE_INVALID 9999 #define POLL_TIME 1000

+enum alg_task_type { + TASK_MIX = 0x0, + TASK_HW, + TASK_INSTR, + TASK_MAX_TYPE, +}; + enum wd_ctx_mode { CTX_MODE_SYNC = 0, CTX_MODE_ASYNC, @@ -130,6 +137,9 @@ struct wd_sched { handle_t h_sched_ctx; };

+typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); +typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); + struct wd_datalist { void *data; __u32 len; diff --git a/include/wd_sched.h b/include/wd_sched.h index 2ae6103..a492d70 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -18,7 +18,11 @@ extern "C" { enum sched_policy_type { /* requests will be sent to ctxs one by one */ SCHED_POLICY_RR = 0, - SCHED_POLICY_BUTT + /* requests will no need ctxs */ + SCHED_POLICY_NONE, + /* requests will need a fixed ctx */ + SCHED_POLICY_SINGLE, + SCHED_POLICY_BUTT, };

struct sched_params { diff --git a/include/wd_util.h b/include/wd_util.h index d4b2814..a730f36 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -117,9 +117,12 @@ struct wd_msg_handle { struct wd_init_attrs { __u32 sched_type; char *alg; + struct wd_alg_driver *driver; struct wd_sched *sched; struct wd_ctx_params *ctx_params; struct wd_ctx_config *ctx_config; + wd_alg_init alg_init; + wd_alg_poll_ctx alg_poll_ctx; };

/* @@ -415,13 +418,69 @@ static inline void wd_alg_clear_init(enum wd_status *status) }

/** - * wd_alg_pre_init() - Request the ctxs and initialize the sched_domain + * wd_ctx_param_init() - Initialize the current device driver according + * to the obtained queue resource and the applied driver. + * @config: device resources requested by the current algorithm. + * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. + * + * Return 0 if succeed and other error number if fail. + */ +int wd_ctx_param_init(struct wd_ctx_params *ctx_params, + struct wd_ctx_params *user_ctx_params, + struct wd_ctx_nums *ctx_set_num, + struct wd_alg_driver *driver, int max_op_type); + +/** + * wd_alg_attrs_init() - Request the ctxs and initialize the sched_domain * with the given devices list, ctxs number and numa mask. * @attrs: the algorithm initialization parameters. * * Return device if succeed and other error number if fail. */ -int wd_alg_pre_init(struct wd_init_attrs *attrs); +int wd_alg_attrs_init(struct wd_init_attrs *attrs); +void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); + +/** + * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain + * with the given devices list, ctxs number and numa mask. + * @task_type: the type of task specified by the current algorithm. + * @alg_name: the name of the algorithm specified by the task. + * + * Return device driver if succeed and other NULL if fail. + */ +struct wd_alg_driver *wd_alg_drv_bind(int task_type, char *alg_name); +void wd_alg_drv_unbind(struct wd_alg_driver *drv); + +/** + * wd_alg_init_driver() - Initialize the current device driver according + * to the obtained queue resource and the applied driver. + * @config: device resources requested by the current algorithm. + * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. + * + * Return 0 if succeed and other error number if fail. + */ +int wd_alg_init_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver, void **drv_priv); +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver, void **drv_priv); + +/** + * wd_dlopen_drv() - Open the dynamic library file of the device driver. + * @cust_lib_dir: the file path of the dynamic library file. + */ +void *wd_dlopen_drv(const char *cust_lib_dir); +void wd_dlclose_drv(void *dlh_list); + +/** + * wd_get_lib_file_path() - Find the path of the dynamic library file in + * the current system. + * @lib_file: the name of the library file. + * @lib_path: the found dynamic library file path. + * @is_dir: Specify whether to query the file dir or the file path. + */ +int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir);

/** * wd_dfx_msg_cnt() - Message counter interface for ctx diff --git a/wd_comp.c b/wd_comp.c index cd938d9..bb142a7 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -230,7 +230,7 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par wd_comp_sched->name = SCHED_RR_NAME; wd_comp_init_attrs.sched = wd_comp_sched;

- ret = wd_alg_pre_init(&wd_comp_init_attrs); + ret = wd_alg_attrs_init(&wd_comp_init_attrs); if (ret) goto out_freesched;

diff --git a/wd_sched.c b/wd_sched.c index 98f4cfd..9a52c4d 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -347,6 +347,82 @@ static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 * return 0; }

+static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) +{ + return (handle_t)0; +} + +static __u32 sched_none_pick_next_ctx(handle_t sched_ctx, + void *sched_key, const int sched_mode) +{ + return 0; +} + +static int sched_none_poll_policy(handle_t h_sched_ctx, + __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + __u32 loop_times = MAX_POLL_TIMES + expect; + __u32 poll_num = 0; + int ret; + + while (loop_times > 0) { + /* Default use ctx 0 */ + ret = sched_ctx->poll_func(0, 1, &poll_num); + if ((ret < 0) && (ret != -EAGAIN)) + return ret; + else if (ret == -EAGAIN) + continue; + + *count += poll_num; + if (*count == expect) + break; + } + + return 0; +} + +static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) +{ + return (handle_t)0; +} + +static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, + void *sched_key, const int sched_mode) +{ +#define CTX_ASYNC 1 +#define CTX_SYNC 0 + + if (sched_mode) + return CTX_ASYNC; + else + return CTX_SYNC; +} + +static int sched_single_poll_policy(handle_t h_sched_ctx, + __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + __u32 loop_times = MAX_POLL_TIMES + expect; + __u32 poll_num = 0; + int ret; + + while (loop_times > 0) { + /* Default async mode use ctx 0 */ + ret = sched_ctx->poll_func(0, 1, &poll_num); + if ((ret < 0) && (ret != -EAGAIN)) + return ret; + else if (ret == -EAGAIN) + continue; + + *count += poll_num; + if (*count == expect) + break; + } + + return 0; +} + static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", @@ -354,7 +430,19 @@ static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { .sched_init = session_sched_init, .pick_next_ctx = session_sched_pick_next_ctx, .poll_policy = session_sched_poll_policy, - }, + }, { + .name = "None scheduler", + .sched_policy = SCHED_POLICY_SINGLE, + .sched_init = sched_none_init, + .pick_next_ctx = sched_none_pick_next_ctx, + .poll_policy = sched_none_poll_policy, + }, { + .name = "Single scheduler", + .sched_policy = SCHED_POLICY_SINGLE, + .sched_init = sched_single_init, + .pick_next_ctx = sched_single_pick_next_ctx, + .poll_policy = sched_single_poll_policy, + } };

static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) @@ -463,9 +551,12 @@ void wd_sched_rr_release(struct wd_sched *sched)

sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; if (!sched_ctx) - goto out; + goto ctx_out;

sched_info = sched_ctx->sched_info; + if (!sched_info) + goto info_out; + for (i = 0; i < sched_ctx->numa_num; i++) { for (j = 0; j < SCHED_MODE_BUTT; j++) { if (sched_info[i].ctx_region[j]) { @@ -475,9 +566,9 @@ void wd_sched_rr_release(struct wd_sched *sched) } }

+info_out: free(sched_ctx); - -out: +ctx_out: free(sched);

return; @@ -531,8 +622,11 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, }

sched->h_sched_ctx = (handle_t)sched_ctx; - sched_info = sched_ctx->sched_info; + if (sched_type == SCHED_POLICY_NONE || + sched_type == SCHED_POLICY_SINGLE) + goto simple_ok;

+ sched_info = sched_ctx->sched_info; for (i = 0; i < numa_num; i++) { for (j = 0; j < SCHED_MODE_BUTT; j++) { sched_info[i].ctx_region[j] = @@ -542,6 +636,7 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, } }

+simple_ok: sched_ctx->poll_func = func; sched_ctx->policy = sched_type; sched_ctx->type_num = type_num; diff --git a/wd_util.c b/wd_util.c index 433dd56..63b51d0 100644 --- a/wd_util.c +++ b/wd_util.c @@ -5,6 +5,8 @@ */

#define _GNU_SOURCE +#include <dirent.h> +#include <dlfcn.h> #include <pthread.h> #include <semaphore.h> #include <string.h> @@ -23,6 +25,8 @@ #define WD_INIT_SLEEP_UTIME 1000 #define WD_INIT_RETRY_TIMES 10000

+#define DEF_DRV_LIB_FILE "libwd.so" + struct msg_pool { /* message array allocated dynamically */ void *msgs; @@ -64,6 +68,72 @@ struct async_task_queue { int (*alg_poll_ctx)(__u32, __u32, __u32 *); };

+struct drv_lib_list { + void *dlhandle; + struct drv_lib_list *next; +}; + +struct acc_alg_item { + char *name; + char *algtype; +}; + +static struct acc_alg_item alg_options[] = { + {"zlib", "zlib-deflate"}, + {"gzip", "gzip"}, + {"deflate", "deflate"}, + {"lz77_zstd", "lz77_zstd"}, + + {"rsa", "rsa"}, + {"dh", "dh"}, + {"ecdh", "ecdh"}, + {"x25519", "x25519"}, + {"x448", "x448"}, + {"ecdsa", "ecdsa"}, + {"sm2", "sm2"}, + + {"ecb(aes)", "cipher"}, + {"cbc(aes)", "cipher"}, + {"xts(aes)", "cipher"}, + {"ofb(aes)", "cipher"}, + {"cfb(aes)", "cipher"}, + {"ctr(aes)", "cipher"}, + {"cbc-cs1(aes)", "cipher"}, + {"cbc-cs2(aes)", "cipher"}, + {"cbc-cs3(aes)", "cipher"}, + {"ecb(sm4)", "cipher"}, + {"xts(sm4)", "cipher"}, + {"cbc(sm4)", "cipher"}, + {"ofb(sm4)", "cipher"}, + {"cfb(sm4)", "cipher"}, + {"ctr(sm4)", "cipher"}, + {"cbc-cs1(sm4)", "cipher"}, + {"cbc-cs2(sm4)", "cipher"}, + {"cbc-cs3(sm4)", "cipher"}, + {"ecb(des)", "cipher"}, + {"cbc(des)", "cipher"}, + {"ecb(des3_ede)", "cipher"}, + {"cbc(des3_ede)", "cipher"}, + + {"ccm(aes)", "aead"}, + {"gcm(aes)", "aead"}, + {"ccm(sm4)", "aead"}, + {"gcm(sm4)", "aead"}, + {"authenc(hmac(sha256),cbc(aes))", "aead"}, + {"authenc(hmac(sha256),cbc(sm4))", "aead"}, + + {"sm3", "digest"}, + {"md5", "digest"}, + {"sha1", "digest"}, + {"sha256", "digest"}, + {"sha224", "digest"}, + {"sha384", "digest"}, + {"sha512", "digest"}, + {"sha512-224", "digest"}, + {"sha512-256", "digest"}, + {"", ""} +}; + static void clone_ctx_to_internal(struct wd_ctx *ctx, struct wd_ctx_internal *ctx_in) { @@ -1779,6 +1849,356 @@ int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) return 0; }

+static void wd_get_alg_type(const char *alg_name, char *alg_type) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(alg_options); i++) { + if (strcmp(alg_name, alg_options[i].name) == 0) { + (void)strcpy(alg_type, alg_options[i].algtype); + break; + } + } +} + +/** + * There are many other .so files in this file directory (/root/lib/), + * and it is necessary to screen out valid uadk driver files + * through this function. + */ +static int file_check_valid(char *lib_file) +{ +#define FILE_HEAD_SZ 4 +#define FILE_TAIL_SZ 4 + int file_len = strlen(lib_file); + char file_head[FILE_HEAD_SZ] = "lib"; + char file_tail[FILE_TAIL_SZ] = ".so"; + int i, j; + + /* Lib file name is libxx_xxx.so */ + for (i = 0; i < FILE_HEAD_SZ - 1; i++) { + if (lib_file[i] != file_head[i]) + return -EINVAL; + } + + for (i = file_len - (FILE_TAIL_SZ - 1), j = 0; + i < file_len && j < FILE_TAIL_SZ; i++, j++) { + if (lib_file[i] != file_tail[j]) + return -EINVAL; + } + + return 0; +} + +static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) +{ + if (!fb_driver->init) { + WD_ERR("soft sec driver have no init interface.\n"); + return -WD_EINVAL; + } + + fb_driver->init(NULL, NULL); + + return 0; +} + +static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) +{ + if (!fb_driver->exit) { + WD_ERR("soft sec driver have no exit interface.\n"); + return; + } + + fb_driver->exit(NULL); +} + +int wd_alg_init_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver, void **drv_priv) +{ + void *priv; + int ret; + + /* Init ctx related resources in specific driver */ + priv = calloc(1, driver->priv_size); + if (!priv) + return -WD_ENOMEM; + + if (!driver->init) { + driver->fallback = 0; + WD_ERR("driver have no init interface.\n"); + ret = -WD_EINVAL; + goto err_alloc; + } + + ret = driver->init(config, priv); + if (ret < 0) { + WD_ERR("driver init failed.\n"); + goto err_alloc; + } + + if (driver->fallback) { + ret = wd_alg_init_fallback((struct wd_alg_driver *)driver->fallback); + if (ret) { + driver->fallback = 0; + WD_ERR("soft alg driver init failed.\n"); + } + } + *drv_priv = priv; + + return 0; + +err_alloc: + free(priv); + return ret; +} + +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver, void **drv_priv) +{ + void *priv = *drv_priv; + + driver->exit(priv); + /* Ctx config just need clear once */ + if (driver->priority == UADK_ALG_HW) + wd_clear_ctx_config(config); + + if (driver->fallback) + wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); + + if (priv) { + free(priv); + priv = NULL; + } +} + +void wd_dlclose_drv(void *dlh_list) +{ + struct drv_lib_list *dlhead = (struct drv_lib_list *)dlh_list; + struct drv_lib_list *dlnode; + + if (!dlhead) { + WD_INFO("driver so file list is empty.\n"); + return; + } + + while (dlhead) { + dlnode = dlhead; + dlhead = dlhead->next; + dlclose(dlnode); + free(dlnode); + } +} + +static void add_lib_to_list(struct drv_lib_list *head, + struct drv_lib_list *node) +{ + struct drv_lib_list *tmp = head; + + while (tmp->next) + tmp = tmp->next; + + tmp->next = node; +} + +int wd_ctx_param_init(struct wd_ctx_params *ctx_params, + struct wd_ctx_params *user_ctx_params, + struct wd_ctx_nums *ctx_set_num, + struct wd_alg_driver *driver, int max_op_type) +{ + int i; + + if (!user_ctx_params) { + ctx_params->bmp = NULL; + ctx_params->ctx_set_num = ctx_set_num; + ctx_params->op_type_num = driver->op_type_num; + if (ctx_params->op_type_num > max_op_type) { + WD_ERR("fail to check driver op type numbers.\n"); + return -WD_EAGAIN; + } + + for (i = 0; i < ctx_params->op_type_num; i++) { + ctx_set_num[i].sync_ctx_num = driver->queue_num; + ctx_set_num[i].async_ctx_num = driver->queue_num; + } + } else { + ctx_params->bmp = user_ctx_params->bmp; + ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; + ctx_params->op_type_num = user_ctx_params->op_type_num; + if (ctx_params->op_type_num > max_op_type) { + WD_ERR("fail to check user op type numbers.\n"); + return -WD_EINVAL; + } + } + + return 0; +} + +static void dladdr_empty(void) {} +int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir) +{ + char file_path[PATH_STR_SIZE] = {0}; + Dl_info file_info; + int len, rc, i; + + /* Get libwd.so file's system path */ + rc = dladdr((void *)dladdr_empty, &file_info); + if (!rc) { + WD_ERR("fail to get lib file path.\n"); + return -WD_EINVAL; + } + strncpy(file_path, file_info.dli_fname, PATH_STR_SIZE - 1); + + /* Clear the file path's tail file name */ + len = strlen(file_path) - 1; + for (i = len; i >= 0; i--) { + if (file_path[i] == '/') { + memset(&file_path[i], 0, PATH_STR_SIZE - i + 1); + break; + } + } + + if (is_dir) { + snprintf(lib_path, PATH_STR_SIZE, "%s", file_path); + return 0; + } + + snprintf(lib_path, PATH_STR_SIZE, "%s/%s", file_path, lib_file); + + return 0; +} + +void *wd_dlopen_drv(const char *cust_lib_dir) +{ + typedef int (*alg_ops)(struct wd_alg_driver *drv); + struct drv_lib_list *node, *head = NULL; + char lib_dir_path[PATH_STR_SIZE] = {0}; + char lib_path[PATH_STR_SIZE] = {0}; + struct dirent *lib_dir; + alg_ops dl_func = NULL; + DIR *wd_dir; + int ret; + + if (!cust_lib_dir) { + ret = wd_get_lib_file_path(NULL, lib_dir_path, true); + if (ret) + return NULL; + } else { + (void)snprintf(lib_path, PATH_STR_SIZE, "%s/%s", cust_lib_dir, DEF_DRV_LIB_FILE); + ret = access(lib_path, F_OK); + if (ret) + return NULL; + + strncpy(lib_dir_path, cust_lib_dir, PATH_STR_SIZE - 1); + } + + wd_dir = opendir(lib_dir_path); + if (!wd_dir) { + WD_ERR("UADK driver lib dir: %s not exist!\n", lib_dir_path); + return NULL; + } + + while ((lib_dir = readdir(wd_dir)) != NULL) { + if (!strncmp(lib_dir->d_name, ".", LINUX_CRTDIR_SIZE) || + !strncmp(lib_dir->d_name, "..", LINUX_PRTDIR_SIZE)) + continue; + + ret = file_check_valid(lib_dir->d_name); + if (ret) + continue; + + node = calloc(1, sizeof(*node)); + if (!node) + goto free_list; + + ret = snprintf(lib_path, PATH_STR_SIZE, "%s/%s", lib_dir_path, lib_dir->d_name); + if (ret < 0) + goto free_node; + + node->dlhandle = dlopen(lib_path, RTLD_NOW); + if (!node->dlhandle) { + free(node); + /* there are many other files need to skip */ + continue; + } + + dl_func = dlsym(node->dlhandle, "wd_alg_driver_register"); + if (dl_func == NULL) { + dlclose(node->dlhandle); + free(node); + continue; + } + + if (!head) + head = node; + else + add_lib_to_list(head, node); + } + closedir(wd_dir); + + return (void *)head; + +free_node: + free(node); +free_list: + closedir(wd_dir); + wd_dlclose_drv(head); + return NULL; +} + +struct wd_alg_driver *wd_alg_drv_bind(int task_type, char *alg_name) +{ + struct wd_alg_driver *set_driver = NULL; + struct wd_alg_driver *drv; + + /* Get alg driver and dev name */ + switch (task_type) { + case TASK_INSTR: + drv = wd_request_drv(alg_name, true); + if (!drv) { + WD_ERR("no soft %s driver support\n", alg_name); + return NULL; + } + set_driver = drv; + set_driver->fallback = 0; + break; + case TASK_HW: + case TASK_MIX: + drv = wd_request_drv(alg_name, false); + if (!drv) { + WD_ERR("no HW %s driver support\n", alg_name); + return NULL; + } + set_driver = drv; + set_driver->fallback = 0; + if (task_type == TASK_MIX) { + drv = wd_request_drv(alg_name, true); + if (!drv) { + set_driver->fallback = 0; + WD_ERR("no soft %s driver support\n", alg_name); + } else { + set_driver->fallback = (handle_t)drv; + WD_ERR("successful to get soft driver\n"); + } + } + break; + } + + return set_driver; +} + +void wd_alg_drv_unbind(struct wd_alg_driver *drv) +{ + struct wd_alg_driver *fb_drv = NULL; + + if (!drv) + return; + + fb_drv = (struct wd_alg_driver *)drv->fallback; + if (fb_drv) + wd_release_drv(fb_drv); + wd_release_drv(drv); +} + bool wd_alg_try_init(enum wd_status *status) { enum wd_status expected; @@ -1960,7 +2380,7 @@ free_ctxs: return ret; }

-int wd_alg_pre_init(struct wd_init_attrs *attrs) +static int wd_alg_ctx_init(struct wd_init_attrs *attrs) { struct wd_ctx_config *ctx_config = attrs->ctx_config; struct wd_ctx_params *ctx_params = attrs->ctx_params; @@ -2034,3 +2454,128 @@ out_freelist:

return ret; } + +static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) +{ + int i; + + for (i = 0; i < ctx_config->ctx_num; i++) + if (ctx_config->ctxs[i].ctx) { + wd_release_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + + free(ctx_config->ctxs); +} + +int wd_alg_attrs_init(struct wd_init_attrs *attrs) +{ + wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; + wd_alg_init alg_init_func = attrs->alg_init; + struct wd_alg_driver *driver = attrs->driver; + __u32 sched_type = attrs->sched_type; + char *alg = attrs->alg; + struct wd_ctx_config *ctx_config = NULL; + struct wd_sched *alg_sched = NULL; + char alg_type[WD_NAME_SIZE]; + __u32 op_type_num; + int ret; + + if (!attrs->ctx_params) + return -WD_EINVAL; + + op_type_num = attrs->ctx_params->op_type_num; + + switch (driver->priority) { + case UADK_ALG_SOFT: + case UADK_ALG_CE_INSTR: + /* No need to alloc resource */ + if (sched_type != SCHED_POLICY_NONE) + return -WD_EINVAL; + + alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); + if (!alg_sched) { + WD_ERR("fail to alloc scheduler\n"); + return -WD_EINVAL; + } + attrs->sched = alg_sched; + + ret = wd_sched_rr_instance(alg_sched, NULL); + if (ret) { + WD_ERR("fail to instance scheduler\n"); + goto out_freesched; + } + break; + case UADK_ALG_SVE_INSTR: + /* Todo lock cpu core */ + if (sched_type != SCHED_POLICY_SINGLE) + return -WD_EINVAL; + + alg_sched = wd_sched_rr_alloc(SCHED_POLICY_SINGLE, 1, 1, alg_poll_func); + if (!alg_sched) { + WD_ERR("fail to alloc scheduler\n"); + return -WD_EINVAL; + } + attrs->sched = alg_sched; + + ret = wd_sched_rr_instance(alg_sched, NULL); + if (ret) { + WD_ERR("fail to instance scheduler\n"); + goto out_freesched; + } + break; + case UADK_ALG_HW: + wd_get_alg_type(alg, alg_type); + attrs->alg = alg_type; + + ctx_config = calloc(1, sizeof(*ctx_config)); + if (!ctx_config) { + WD_ERR("fail to alloc ctx config\n"); + return -WD_ENOMEM; + } + attrs->ctx_config = ctx_config; + + alg_sched = wd_sched_rr_alloc(sched_type, op_type_num, + numa_max_node() + 1, alg_poll_func); + if (!alg_sched) { + WD_ERR("fail to instance scheduler\n"); + ret = -WD_EINVAL; + goto out_ctx_config; + } + attrs->sched = alg_sched; + + ret = wd_alg_ctx_init(attrs); + if (ret) { + WD_ERR("fail to init ctx\n"); + goto out_freesched; + } + + ret = alg_init_func(ctx_config, alg_sched); + if (ret) + goto out_pre_init; + } + + return 0; + +out_pre_init: + wd_alg_ctx_uninit(ctx_config); +out_freesched: + wd_sched_rr_release(alg_sched); +out_ctx_config: + if (ctx_config) + free(ctx_config); + return ret; +} + +void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) +{ + struct wd_ctx_config *ctx_config = attrs->ctx_config; + struct wd_sched *alg_sched = attrs->sched; + + if (ctx_config) { + wd_alg_ctx_uninit(ctx_config); + free(ctx_config); + } + wd_sched_rr_release(alg_sched); +} +

1. Add the init2 interface for the cipher algorithm to simplify the user's use of the cipher algorithm.

2. After adding the cipher module of the init2 interface, combine it Dynamically load the initialization part, transform HiSilicon sec driven, and implemented using the dynamic loading function Connection between driver and algorithm layer.

Signed-off-by: Longfang Liu liulongfang@huawei.com --- drv/hisi_sec.c | 102 ++++++++++-- include/drv/wd_aead_drv.h | 2 +- include/drv/wd_cipher_drv.h | 26 --- include/drv/wd_digest_drv.h | 2 +- libwd_crypto.map | 3 + wd_cipher.c | 305 +++++++++++++++++++++++++++--------- 6 files changed, 325 insertions(+), 115 deletions(-)

diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index c30b653..5e7da98 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -80,6 +80,8 @@ #define WD_CIPHER_THEN_DIGEST 0x0 #define WD_DIGEST_THEN_CIPHER 0x1

+#define SEC_CTX_Q_NUM_DEF 1 + enum C_ALG { C_ALG_DES = 0x0, C_ALG_3DES = 0x1, @@ -508,9 +510,54 @@ static __u32 g_sec_hmac_full_len[WD_DIGEST_TYPE_MAX] = { SEC_HMAC_SHA512_MAC_LEN, SEC_HMAC_SHA512_224_MAC_LEN, SEC_HMAC_SHA512_256_MAC_LEN };

-int hisi_sec_init(struct wd_ctx_config_internal *config, void *priv); +int hisi_sec_init(void *conf, void *priv); void hisi_sec_exit(void *priv);

+static int hisi_sec_get_usage(void *param) +{ + return 0; +} + +#define GEN_SEC_ALG_DRIVER(sec_alg_name) \ +{\ + .drv_name = "hisi_sec2",\ + .alg_name = sec_alg_name,\ + .priority = UADK_ALG_HW,\ + .priv_size = sizeof(struct hisi_sec_ctx),\ + .queue_num = SEC_CTX_Q_NUM_DEF,\ + .op_type_num = 1,\ + .fallback = 0,\ + .init = hisi_sec_init,\ + .exit = hisi_sec_exit,\ + .get_usage = hisi_sec_get_usage,\ +} + +static struct wd_alg_driver cipher_alg_driver[] = { + GEN_SEC_ALG_DRIVER("ecb(aes)"), + GEN_SEC_ALG_DRIVER("cbc(aes)"), + GEN_SEC_ALG_DRIVER("xts(aes)"), + GEN_SEC_ALG_DRIVER("ecb(sm4)"), + GEN_SEC_ALG_DRIVER("cbc(sm4)"), + GEN_SEC_ALG_DRIVER("ctr(sm4)"), + GEN_SEC_ALG_DRIVER("xts(sm4)"), + GEN_SEC_ALG_DRIVER("ecb(des)"), + GEN_SEC_ALG_DRIVER("cbc(des)"), + GEN_SEC_ALG_DRIVER("ecb(des3_ede)"), + GEN_SEC_ALG_DRIVER("cbc(des3_ede)"), + + GEN_SEC_ALG_DRIVER("ctr(aes)"), + GEN_SEC_ALG_DRIVER("ofb(aes)"), + GEN_SEC_ALG_DRIVER("cfb(aes)"), + GEN_SEC_ALG_DRIVER("cbc-cs1(aes)"), + GEN_SEC_ALG_DRIVER("cbc-cs2(aes)"), + GEN_SEC_ALG_DRIVER("cbc-cs3(aes)"), + GEN_SEC_ALG_DRIVER("ofb(sm4)"), + GEN_SEC_ALG_DRIVER("cfb(sm4)"), + GEN_SEC_ALG_DRIVER("cbc-cs1(sm4)"), + GEN_SEC_ALG_DRIVER("cbc-cs2(sm4)"), + GEN_SEC_ALG_DRIVER("cbc-cs3(sm4)"), +}; + static void dump_sec_msg(void *msg, const char *alg) { struct wd_cipher_msg *cmsg; @@ -1064,16 +1111,6 @@ int hisi_sec_cipher_recv(handle_t ctx, void *cipher_msg) return 0; }

-static struct wd_cipher_driver hisi_cipher_driver = { - .drv_name = "hisi_sec2", - .alg_name = "cipher", - .drv_ctx_size = sizeof(struct hisi_sec_ctx), - .init = hisi_sec_init, - .exit = hisi_sec_exit, -}; - -WD_CIPHER_SET_DRIVER(hisi_cipher_driver); - static int fill_cipher_bd3_alg(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) { @@ -2526,11 +2563,15 @@ int hisi_sec_aead_recv_v3(handle_t ctx, void *aead_msg) static void hisi_sec_driver_adapter(struct hisi_qp *qp) { struct hisi_qm_queue_info q_info = qp->q_info; + int alg_num, i;

if (q_info.hw_type == HISI_QM_API_VER2_BASE) { WD_ERR("hisi sec init Kunpeng920!\n"); - hisi_cipher_driver.cipher_send = hisi_sec_cipher_send; - hisi_cipher_driver.cipher_recv = hisi_sec_cipher_recv; + alg_num = ARRAY_SIZE(cipher_alg_driver); + for (i = 0; i < alg_num; i++) { + cipher_alg_driver[i].send = hisi_sec_cipher_send; + cipher_alg_driver[i].recv = hisi_sec_cipher_recv; + }

hisi_digest_driver.digest_send = hisi_sec_digest_send; hisi_digest_driver.digest_recv = hisi_sec_digest_recv; @@ -2539,8 +2580,11 @@ static void hisi_sec_driver_adapter(struct hisi_qp *qp) hisi_aead_driver.aead_recv = hisi_sec_aead_recv; } else { WD_ERR("hisi sec init Kunpeng930!\n"); - hisi_cipher_driver.cipher_send = hisi_sec_cipher_send_v3; - hisi_cipher_driver.cipher_recv = hisi_sec_cipher_recv_v3; + alg_num = ARRAY_SIZE(cipher_alg_driver); + for (i = 0; i < alg_num; i++) { + cipher_alg_driver[i].send = hisi_sec_cipher_send_v3; + cipher_alg_driver[i].recv = hisi_sec_cipher_recv_v3; + }

hisi_digest_driver.digest_send = hisi_sec_digest_send_v3; hisi_digest_driver.digest_recv = hisi_sec_digest_recv_v3; @@ -2550,8 +2594,9 @@ static void hisi_sec_driver_adapter(struct hisi_qp *qp) } }

-int hisi_sec_init(struct wd_ctx_config_internal *config, void *priv) +int hisi_sec_init(void *conf, void *priv) { + struct wd_ctx_config_internal *config = conf; struct hisi_sec_ctx *sec_ctx = priv; struct hisi_qm_priv qm_priv; handle_t h_qp = 0; @@ -2609,3 +2654,28 @@ void hisi_sec_exit(void *priv) hisi_qm_free_qp(h_qp); } } + +static void __attribute__((constructor)) hisi_sec2_probe(void) +{ + int alg_num = ARRAY_SIZE(cipher_alg_driver); + int i, ret; + + WD_INFO("Info: register SEC alg drivers!\n"); + + for (i = 0; i < alg_num; i++) { + ret = wd_alg_driver_register(&cipher_alg_driver[i]); + if (ret) + WD_ERR("Error: register SEC %s failed!\n", + cipher_alg_driver[i].alg_name); + } +} + +static void __attribute__((destructor)) hisi_sec2_remove(void) +{ + int alg_num = ARRAY_SIZE(cipher_alg_driver); + int i; + + for (i = 0; i < alg_num; i++) + wd_alg_driver_unregister(&cipher_alg_driver[i]); +} + diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 7c657f6..8446238 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -67,7 +67,7 @@ struct wd_aead_driver { const char *drv_name; const char *alg_name; __u32 drv_ctx_size; - int (*init)(struct wd_ctx_config_internal *config, void *priv); + int (*init)(void *conf, void *priv); void (*exit)(void *priv); int (*aead_send)(handle_t ctx, void *aead_msg); int (*aead_recv)(handle_t ctx, void *aead_msg); diff --git a/include/drv/wd_cipher_drv.h b/include/drv/wd_cipher_drv.h index 82fb89a..c6d8ddf 100644 --- a/include/drv/wd_cipher_drv.h +++ b/include/drv/wd_cipher_drv.h @@ -50,34 +50,8 @@ struct wd_cipher_msg { __u8 *out; };

-struct wd_cipher_driver { - const char *drv_name; - const char *alg_name; - __u32 drv_ctx_size; - int (*init)(struct wd_ctx_config_internal *config, void *priv); - void (*exit)(void *priv); - int (*cipher_send)(handle_t ctx, void *cipher_msg); - int (*cipher_recv)(handle_t ctx, void *cipher_msg); -}; - -void wd_cipher_set_driver(struct wd_cipher_driver *drv); -struct wd_cipher_driver *wd_cipher_get_driver(void); struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag);

-#ifdef WD_STATIC_DRV -#define WD_CIPHER_SET_DRIVER(drv) \ -struct wd_cipher_driver *wd_cipher_get_driver(void) \ -{ \ - return &drv; \ -} -#else -#define WD_CIPHER_SET_DRIVER(drv) \ -static void __attribute__((constructor)) set_cipher_driver(void) \ -{ \ - wd_cipher_set_driver(&(drv)); \ -} -#endif - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index 586588b..96b32e2 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -55,7 +55,7 @@ struct wd_digest_driver { const char *drv_name; const char *alg_name; __u32 drv_ctx_size; - int (*init)(struct wd_ctx_config_internal *config, void *priv); + int (*init)(void *conf, void *priv); void (*exit)(void *priv); int (*digest_send)(handle_t ctx, void *digest_msg); int (*digest_recv)(handle_t ctx, void *digest_msg); diff --git a/libwd_crypto.map b/libwd_crypto.map index 5fadc53..f7d1aa0 100644 --- a/libwd_crypto.map +++ b/libwd_crypto.map @@ -2,6 +2,9 @@ UADK_CRYPTO_2.0 { global: wd_cipher_init; wd_cipher_uninit; + wd_cipher_init2; + wd_cipher_init2_; + wd_cipher_uninit2; wd_cipher_alloc_sess; wd_cipher_free_sess; wd_cipher_set_key; diff --git a/wd_cipher.c b/wd_cipher.c index b627c0d..236814e 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -43,15 +43,24 @@ static const unsigned char des_weak_keys[DES_WEAK_KEY_NUM][DES_KEY_SIZE] = { {0xFE, 0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1} };

+static char *wd_cipher_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] = { + {"ecb(sm4)", "cbc(sm4)", "ctr(sm4)", "xts(sm4)", "ofb(sm4)", + "cfb(sm4)", "cbc-cs1(sm4)", "cbc-cs2(sm4)", "cbc-cs3(sm4)"}, + {"ecb(aes)", "cbc(aes)", "ctr(aes)", "xts(aes)", "ofb(aes)", + "cfb(aes)", "cbc-cs1(aes)", "cbc-cs2(aes)", "cbc-cs3(aes)"}, + {"cbc(des)", "ecb(des)",}, + {"cbc(des3_ede)", "ecb(des3_ede)",} +}; + struct wd_cipher_setting { enum wd_status status; struct wd_ctx_config_internal config; struct wd_sched sched; - void *sched_ctx; - struct wd_cipher_driver *driver; + struct wd_async_msg_pool pool; + struct wd_alg_driver *driver; void *priv; void *dlhandle; - struct wd_async_msg_pool pool; + void *dlh_list; } wd_cipher_setting;

struct wd_cipher_sess { @@ -67,32 +76,51 @@ struct wd_cipher_sess { };

struct wd_env_config wd_cipher_env_config; +static struct wd_init_attrs wd_cipher_init_attrs;

-#ifdef WD_STATIC_DRV -static void wd_cipher_set_static_drv(void) +static void wd_cipher_close_driver(void) { - wd_cipher_setting.driver = wd_cipher_get_driver(); - if (!wd_cipher_setting.driver) - WD_ERR("failed to get driver!\n"); + if (wd_cipher_setting.dlhandle) { + wd_release_drv(wd_cipher_setting.driver); + dlclose(wd_cipher_setting.dlhandle); + wd_cipher_setting.dlhandle = NULL; + } } -#else -static void __attribute__((constructor)) wd_cipher_open_driver(void) + +static int wd_cipher_open_driver(void) { - wd_cipher_setting.dlhandle = dlopen("libhisi_sec.so", RTLD_NOW); - if (!wd_cipher_setting.dlhandle) + struct wd_alg_driver *driver = NULL; + const char *alg_name = "cbc(aes)"; + char lib_path[PATH_STR_SIZE]; + int ret; + + /* + * Compatible with the normal acquisition of device + * drivers in the init interface + */ + if (wd_cipher_setting.dlh_list) + return 0; + + ret = wd_get_lib_file_path("libhisi_sec.so", lib_path, false); + if (ret) + return ret; + + wd_cipher_setting.dlhandle = dlopen(lib_path, RTLD_NOW); + if (!wd_cipher_setting.dlhandle) { WD_ERR("failed to open libhisi_sec.so, %s\n", dlerror()); -} + return -WD_EINVAL; + }

-static void __attribute__((destructor)) wd_cipher_close_driver(void) -{ - if (wd_cipher_setting.dlhandle) - dlclose(wd_cipher_setting.dlhandle); -} -#endif + driver = wd_request_drv(alg_name, false); + if (!driver) { + wd_cipher_close_driver(); + WD_ERR("failed to get %s driver support\n", alg_name); + return -WD_EINVAL; + }

-void wd_cipher_set_driver(struct wd_cipher_driver *drv) -{ - wd_cipher_setting.driver = drv; + wd_cipher_setting.driver = driver; + + return 0; }

static bool is_des_weak_key(const __u8 *key) @@ -185,6 +213,7 @@ int wd_cipher_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { struct wd_cipher_sess *sess = NULL; + bool ret;

if (unlikely(!setup)) { WD_ERR("invalid: cipher input setup is NULL!\n"); @@ -198,18 +227,35 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) } memset(sess, 0, sizeof(struct wd_cipher_sess));

+ if (setup->alg >= WD_CIPHER_ALG_TYPE_MAX || + setup->mode >= WD_CIPHER_MODE_TYPE_MAX) { + WD_ERR("failed to check algorithm!\n"); + return (handle_t)0; + } + sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; sess->alg = setup->alg; sess->mode = setup->mode; + ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + if (!ret) { + WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); + goto err_sess; + } + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_cipher_setting.sched.sched_init( wd_cipher_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - free(sess); - return (handle_t)0; + goto err_sess; }

return (handle_t)sess; + +err_sess: + if (sess->sched_key) + free(sess->sched_key); + free(sess); + return (handle_t)0; }

void wd_cipher_free_sess(handle_t h_sess) @@ -228,38 +274,24 @@ void wd_cipher_free_sess(handle_t h_sess) free(sess); }

-int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_cipher_common_init(struct wd_ctx_config *config, + struct wd_sched *sched) { - void *priv; - bool flag; int ret;

- flag = wd_alg_try_init(&wd_cipher_setting.status); - if (!flag) - return 0; - - ret = wd_init_param_check(config, sched); - if (ret) - goto out_clear_init; - ret = wd_set_epoll_en("WD_CIPHER_EPOLL_EN", &wd_cipher_setting.config.epoll_en); if (ret < 0) - goto out_clear_init; + return ret;

ret = wd_init_ctx_config(&wd_cipher_setting.config, config); if (ret < 0) - goto out_clear_init; + return ret;

ret = wd_init_sched(&wd_cipher_setting.sched, sched); if (ret < 0) goto out_clear_ctx_config;

-#ifdef WD_STATIC_DRV - /* set driver */ - wd_cipher_set_static_drv(); -#endif - /* allocate async pool for every ctx */ ret = wd_init_async_request_pool(&wd_cipher_setting.pool, config->ctx_num, WD_POOL_MAX_ENTRIES, @@ -267,52 +299,183 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret < 0) goto out_clear_sched;

- /* init ctx related resources in specific driver */ - priv = calloc(1, wd_cipher_setting.driver->drv_ctx_size); - if (!priv) { - ret = -WD_ENOMEM; + ret = wd_alg_init_driver(&wd_cipher_setting.config, + wd_cipher_setting.driver, + &wd_cipher_setting.priv); + if (ret) goto out_clear_pool; - } - wd_cipher_setting.priv = priv; - - ret = wd_cipher_setting.driver->init(&wd_cipher_setting.config, priv); - if (ret < 0) { - WD_ERR("failed to do dirver init, ret = %d.\n", ret); - goto out_free_priv; - } - - wd_alg_set_init(&wd_cipher_setting.status);

return 0;

-out_free_priv: - free(priv); - wd_cipher_setting.priv = NULL; out_clear_pool: wd_uninit_async_request_pool(&wd_cipher_setting.pool); out_clear_sched: wd_clear_sched(&wd_cipher_setting.sched); out_clear_ctx_config: wd_clear_ctx_config(&wd_cipher_setting.config); -out_clear_init: - wd_alg_clear_init(&wd_cipher_setting.status); return ret; }

-void wd_cipher_uninit(void) +static void wd_cipher_common_uninit(void) { void *priv = wd_cipher_setting.priv;

if (!priv) return;

- wd_cipher_setting.driver->exit(priv); - wd_cipher_setting.priv = NULL; - free(priv); - + /* uninit async request pool */ wd_uninit_async_request_pool(&wd_cipher_setting.pool); + + /* unset config, sched, driver */ wd_clear_sched(&wd_cipher_setting.sched); - wd_clear_ctx_config(&wd_cipher_setting.config); + + wd_alg_uninit_driver(&wd_cipher_setting.config, + wd_cipher_setting.driver, &priv); +} + +int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) +{ + enum wd_status status; + bool flag; + int ret; + + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status == WD_INIT) { + WD_INFO("UADK cipher has been initialized with wd_cipher_init2()!\n"); + return 0; + } + + flag = wd_alg_try_init(&wd_cipher_setting.status); + if (!flag) + return 0; + + ret = wd_init_param_check(config, sched); + if (ret) + goto out_clear_init; + + ret = wd_cipher_open_driver(); + if (ret) + goto out_clear_init; + + ret = wd_cipher_common_init(config, sched); + if (ret) + goto out_close_driver; + + wd_alg_set_init(&wd_cipher_setting.status); + + return 0; + +out_close_driver: + wd_cipher_close_driver(); +out_clear_init: + wd_alg_clear_init(&wd_cipher_setting.status); + return ret; +} + +void wd_cipher_uninit(void) +{ + wd_cipher_common_uninit(); + + wd_cipher_close_driver(); + wd_alg_clear_init(&wd_cipher_setting.status); +} + +int wd_cipher_init2(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) +{ + struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; + struct wd_ctx_params cipher_ctx_params = {0}; + enum wd_status status; + int ret = 0; + bool flag; + + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status == WD_INIT) { + WD_INFO("UADK cipher has been initialized with wd_cipher_init()!\n"); + return 0; + } + + flag = wd_alg_try_init(&wd_cipher_setting.status); + if (!flag) + return 0; + + if (!alg || sched_type > SCHED_POLICY_BUTT || + task_type < 0 || task_type > TASK_MAX_TYPE) { + WD_ERR("invalid: input param is wrong!\n"); + ret = -WD_EINVAL; + goto out_uninit; + } + + /* + * Driver lib file path could set by env param. + * than open tham by wd_dlopen_drv() + * use NULL means dynamic query path + */ + wd_cipher_setting.dlh_list = wd_dlopen_drv(NULL); + if (!wd_cipher_setting.dlh_list) { + WD_ERR("fail to open driver lib files.\n"); + goto out_uninit; + } + +res_retry: + memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + + /* Get alg driver and dev name */ + wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); + if (!wd_cipher_setting.driver) { + WD_ERR("fail to bind a valid driver.\n"); + goto out_dlopen; + } + + ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, + cipher_ctx_num, wd_cipher_setting.driver, + WD_CIPHER_DECRYPTION + 1); + if (ret) { + if (ret == -WD_EAGAIN) { + wd_disable_drv(wd_cipher_setting.driver); + wd_alg_drv_unbind(wd_cipher_setting.driver); + goto res_retry; + } + goto out_driver; + } + + wd_cipher_init_attrs.alg = alg; + wd_cipher_init_attrs.sched_type = sched_type; + wd_cipher_init_attrs.driver = wd_cipher_setting.driver; + wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; + wd_cipher_init_attrs.alg_init = wd_cipher_common_init; + wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; + ret = wd_alg_attrs_init(&wd_cipher_init_attrs); + if (ret) { + if (ret == -WD_ENODEV) { + wd_disable_drv(wd_cipher_setting.driver); + wd_alg_drv_unbind(wd_cipher_setting.driver); + goto res_retry; + } + WD_ERR("fail to init alg attrs.\n"); + goto out_driver; + } + + wd_alg_set_init(&wd_cipher_setting.status); + + return 0; + +out_driver: + wd_alg_drv_unbind(wd_cipher_setting.driver); +out_dlopen: + wd_dlclose_drv(wd_cipher_setting.dlh_list); +out_uninit: + wd_alg_clear_init(&wd_cipher_setting.status); + return ret; +} + +void wd_cipher_uninit2(void) +{ + wd_cipher_common_uninit(); + + wd_alg_attrs_uninit(&wd_cipher_init_attrs); + wd_alg_drv_unbind(wd_cipher_setting.driver); + wd_dlclose_drv(wd_cipher_setting.dlh_list); + wd_cipher_setting.dlh_list = NULL; wd_alg_clear_init(&wd_cipher_setting.status); }

@@ -419,8 +582,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_msg_handle msg_handle; int ret;

- msg_handle.send = wd_cipher_setting.driver->cipher_send; - msg_handle.recv = wd_cipher_setting.driver->cipher_recv; + msg_handle.send = wd_cipher_setting.driver->send; + msg_handle.recv = wd_cipher_setting.driver->recv;

pthread_spin_lock(&ctx->lock); ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, @@ -499,7 +662,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id;

- ret = wd_cipher_setting.driver->cipher_send(ctx->ctx, msg); + ret = wd_cipher_setting.driver->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -547,7 +710,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx;

do { - ret = wd_cipher_setting.driver->cipher_recv(ctx->ctx, &resp_msg); + ret = wd_cipher_setting.driver->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) {