Added heterogeneous scheduling function in UADK. Combined hard computing acceleration and soft computing instruction acceleration functions,. keeping both types of acceleration functions effective at the same time. This improves acceleration capability.
Signed-off-by: Longfang Liu liulongfang@huawei.com --- include/wd_alg.h | 16 +- include/wd_alg_common.h | 39 +- include/wd_sched.h | 4 + include/wd_util.h | 28 +- wd_util.c | 858 +++++++++++++++++++++++++++++----------- 5 files changed, 704 insertions(+), 241 deletions(-)
diff --git a/include/wd_alg.h b/include/wd_alg.h index 6203893..e5ae9af 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -63,13 +63,22 @@ extern "C" { #endif
enum alg_priority { - UADK_ALG_SOFT = 0x0, + UADK_ALG_HW = 0x0, UADK_ALG_CE_INSTR = 0x1, UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_SOFT = 0x3 };
-/* +enum alg_drv_type { + ALG_DRV_HW = 0x0, + ALG_DRV_CE_INS, + ALG_DRV_SVE_INS, + ALG_DRV_SOFT, + ALG_DRV_INS, + ALG_DRV_FB, +}; + +/** * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver * @priority: priority of the type of algorithm supported by the driver @@ -106,6 +115,7 @@ struct wd_alg_driver { int op_type_num; int priv_size; handle_t fallback; + int init_state;
int (*init)(void *conf, void *priv); void (*exit)(void *priv); diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index d1c0038..5b973f0 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -44,6 +44,7 @@ extern "C" {
/* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define MAX_SOFT_QUEUE_LENGTH 1024U
enum alg_task_type { TASK_MIX = 0x0, @@ -68,6 +69,26 @@ enum wd_init_type { WD_TYPE_V2, };
+struct wd_soft_sqe { + __u8 used; + __u8 result; + __u8 complete; + __u32 id; +}; + +/** + * default queue length set to 1024 + */ +struct wd_soft_ctx { + pthread_spinlock_t slock; + __u32 head; + struct wd_soft_sqe qfifo[MAX_SOFT_QUEUE_LENGTH]; + pthread_spinlock_t rlock; + __u32 tail; + __u32 run_num; + void *priv; +}; + /** * struct wd_ctx - Define one ctx and related type. * @ctx: The ctx itself. @@ -80,6 +101,7 @@ struct wd_ctx { handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; };
/** @@ -123,6 +145,15 @@ struct wd_ctx_nums { __u32 async_ctx_num; };
+/* 0x0 mean calloc init value */ +enum wd_ctx_priority { + UADK_CTX_HW = 0x0, + UADK_CTX_CE_INS = 0x1, + UADK_CTX_SVE_INS = 0x2, + UADK_CTX_SOFT = 0x3, + UADK_CTX_ERR +}; + /** * struct wd_ctx_params - Define the ctx sets params which are used for init * algorithms. @@ -140,16 +171,16 @@ struct wd_ctx_params { struct wd_cap_config *cap; };
-struct wd_soft_ctx { - void *priv; -}; - struct wd_ctx_internal { handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + void *drv_priv; + __u32 fb_num; };
struct wd_ctx_config_internal { diff --git a/include/wd_sched.h b/include/wd_sched.h index be541c6..c49b4c9 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -23,6 +23,9 @@ enum sched_policy_type { SCHED_POLICY_NONE, /* requests will need a fixed ctx */ SCHED_POLICY_SINGLE, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + SCHED_POLICY_RTE_LOOP, SCHED_POLICY_BUTT, };
@@ -32,6 +35,7 @@ struct sched_params { __u8 mode; __u32 begin; __u32 end; + int ctx_prop; };
typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/include/wd_util.h b/include/wd_util.h index 63aa5af..9a30dc8 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -125,8 +125,8 @@ struct wd_msg_handle {
struct wd_init_attrs { __u32 sched_type; + __u32 task_type; char *alg; - struct wd_alg_driver *driver; struct wd_sched *sched; struct wd_ctx_params *ctx_params; struct wd_ctx_config *ctx_config; @@ -172,6 +172,9 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in); */ void wd_memset_zero(void *data, __u32 size);
+int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config); +void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config); + /* * wd_init_async_request_pool() - Init async message pools. * @pool: Pointer of message pool. @@ -256,7 +259,7 @@ int wd_check_src_dst(void *src, __u32 in_bytes, void *dst, __u32 out_bytes); * * Return 0 if the datalist is not less than expected size. */ -int wd_check_datalist(struct wd_datalist *head, __u64 size); +int wd_check_datalist(struct wd_datalist *head, __u32 size);
/* @@ -453,8 +456,8 @@ static inline void wd_alg_clear_init(enum wd_status *status) */ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type); + char *alg, int task_type, enum wd_type type, + int max_op_type);
void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params);
@@ -471,12 +474,12 @@ void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); /** * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain * with the given devices list, ctxs number and numa mask. - * @task_type: the type of task specified by the current algorithm. + * @ctx_type: the type of ctx specified by the current algorithm. * @alg_name: the name of the algorithm specified by the task. * * Return device driver if succeed and other NULL if fail. */ -struct wd_alg_driver *wd_alg_drv_bind(int task_type, char *alg_name); +struct wd_alg_driver *wd_alg_drv_bind(__u8 ctx_type, char *alg_name); void wd_alg_drv_unbind(struct wd_alg_driver *drv);
/** @@ -488,10 +491,8 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv); * * Return 0 if succeed and other error number if fail. */ -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv); -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv); +int wd_alg_init_driver(struct wd_ctx_config_internal *config); +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config);
/** * wd_dlopen_drv() - Open the dynamic library file of the device driver. @@ -509,6 +510,9 @@ void wd_dlclose_drv(void *dlh_list); */ int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir);
+int wd_msg_sync_fb(handle_t ctx, void *msg, int err_code); +int wd_msg_recv_fb(handle_t ctx); + /** * wd_dfx_msg_cnt() - Message counter interface for ctx * @config: Ctx configuration in global setting. @@ -552,6 +556,10 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) pthread_spin_unlock(&ctx->lock); }
+int wd_queue_is_busy(struct wd_soft_ctx *sctx); +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); + #ifdef __cplusplus } #endif diff --git a/wd_util.c b/wd_util.c index 2dd81c9..cd92e20 100644 --- a/wd_util.c +++ b/wd_util.c @@ -167,6 +167,7 @@ static void clone_ctx_to_internal(struct wd_ctx *ctx, ctx_in->ctx = ctx->ctx; ctx_in->op_type = ctx->op_type; ctx_in->ctx_mode = ctx->ctx_mode; + ctx_in->ctx_type = ctx->ctx_type; }
static int wd_shm_create(struct wd_ctx_config_internal *in) @@ -239,6 +240,7 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, ret = -WD_EINVAL; goto err_out; } + clone_ctx_to_internal(cfg->ctxs + i, ctxs + i); ret = pthread_spin_init(&ctxs[i].lock, PTHREAD_PROCESS_SHARED); if (ret) { @@ -378,7 +380,6 @@ int wd_init_async_request_pool(struct wd_async_msg_pool *pool, struct wd_ctx_con int ret;
pool->pool_num = pool_num; - pool->pools = calloc(1, pool_num * sizeof(struct msg_pool)); if (!pool->pools) { WD_ERR("failed to alloc memory for async msg pools!\n"); @@ -420,15 +421,8 @@ void wd_uninit_async_request_pool(struct wd_async_msg_pool *pool) void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, int ctx_idx, __u32 tag) { - struct msg_pool *p; - __u32 msg_num; - - if ((__u32)ctx_idx > pool->pool_num) { - WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); - return NULL; - } - p = &pool->pools[ctx_idx]; - msg_num = p->msg_num; + struct msg_pool *p = &pool->pools[ctx_idx]; + __u32 msg_num = p->msg_num;
/* tag value start from 1 */ if (tag == 0 || tag > msg_num) { @@ -487,10 +481,10 @@ int wd_check_src_dst(void *src, __u32 in_bytes, void *dst, __u32 out_bytes) return 0; }
-int wd_check_datalist(struct wd_datalist *head, __u64 size) +int wd_check_datalist(struct wd_datalist *head, __u32 size) { struct wd_datalist *tmp = head; - __u64 list_size = 0; + __u32 list_size = 0;
while (tmp) { if (tmp->data) @@ -1026,7 +1020,7 @@ static int wd_parse_env(struct wd_env_config *config) if (!var_s || !strlen(var_s)) { var_s = var->def_val; WD_INFO("no %s environment variable! Use default: %s\n", - var->name, var->def_val); + var->name, var->def_val); }
ret = var->parse_fn(config, var_s); @@ -1224,7 +1218,6 @@ err_free_ctxs: free(ctx_config->ctxs); err_free_ctx_config: free(ctx_config); - config->ctx_config = NULL; return ret; }
@@ -1479,7 +1472,6 @@ static void *async_poll_process_func(void *args) goto out; } out: - pthread_exit(NULL); return NULL; }
@@ -1794,7 +1786,7 @@ int wd_check_ctx(struct wd_ctx_config_internal *config, __u8 mode, __u32 idx) }
ctx = config->ctxs + idx; - if (ctx->ctx_mode != mode) { + if (ctx->ctx_type == UADK_CTX_HW && ctx->ctx_mode != mode) { WD_ERR("invalid: ctx(%u) mode is %hhu!\n", idx, ctx->ctx_mode); return -WD_EINVAL; } @@ -1825,6 +1817,37 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en) return 0; }
+int wd_msg_sync_fb(handle_t ctx, void *msg, int err_code) +{ + struct wd_ctx_internal *drv_ctx = (struct wd_ctx_internal *)ctx; + struct wd_alg_driver *fb_drv = NULL; + int ret; + + // Swith to fallback; + if (!drv_ctx->drv || !drv_ctx->drv->fallback) + return err_code; + fb_drv = (struct wd_alg_driver *)drv_ctx->drv->fallback; + ret = fb_drv->send(ctx, msg); + + return ret; +} + +int wd_msg_recv_fb(handle_t ctx) +{ + struct wd_ctx_internal *drv_ctx = (struct wd_ctx_internal *)ctx; + __u32 fb_recv_num; + + // Check fallback; + if (!drv_ctx->drv || !drv_ctx->drv->fallback) + return 0; + + fb_recv_num = __atomic_load_n(&drv_ctx->fb_num, __ATOMIC_ACQUIRE); + __atomic_fetch_sub(&drv_ctx->fb_num, fb_recv_num, __ATOMIC_ACQUIRE); + + return fb_recv_num; +} + + int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, void *msg, __u64 *balance, bool epoll_en) { @@ -1838,7 +1861,7 @@ int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, ret = msg_handle->send(ctx, msg); if (unlikely(ret < 0)) { WD_ERR("failed to send msg to hw, ret = %d!\n", ret); - return ret; + goto fb_cal; }
do { @@ -1852,7 +1875,7 @@ int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, if (ret != -WD_EAGAIN) { if (unlikely(ret < 0)) { WD_ERR("failed to recv msg: error = %d!\n", ret); - return ret; + goto fb_cal; } break; } @@ -1869,6 +1892,11 @@ int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx,
if (balance) *balance = rx_cnt; + return ret; + +fb_cal: + // Swith to fallback; + ret = wd_msg_sync_fb(ctx, msg, ret);
return ret; } @@ -1974,6 +2002,7 @@ static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) return -WD_EINVAL; }
+ WD_ERR("debug: call function: %s!\n", __func__); fb_driver->init(NULL, NULL);
return 0; @@ -1989,12 +2018,20 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) fb_driver->exit(NULL); }
-int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv) +static int wd_ctx_init_driver(struct wd_ctx_config_internal *config, + struct wd_ctx_internal *ctx_config) { - void *priv; + struct wd_alg_driver *driver = ctx_config->drv; + void *priv = ctx_config->drv_priv; int ret;
+ if (!driver) + return 0; + + /* Prevent repeated initialization */ + if (driver->init_state) + return 0; + if (!driver->priv_size) { WD_ERR("invalid: driver priv ctx size is zero!\n"); return -WD_EINVAL; @@ -2017,6 +2054,7 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("driver init failed.\n"); goto err_alloc; } + driver->init_state = 1;
if (driver->fallback) { ret = wd_alg_init_fallback((struct wd_alg_driver *)driver->fallback); @@ -2025,21 +2063,30 @@ int wd_alg_init_driver(struct wd_ctx_config_internal *config, WD_ERR("soft alg driver init failed.\n"); } } - *drv_priv = priv;
return 0;
err_alloc: free(priv); + ctx_config->drv_priv = NULL; return ret; }
-void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver, void **drv_priv) +static void wd_ctx_uninit_driver(struct wd_ctx_config_internal *config, + struct wd_ctx_internal *ctx_config) { - void *priv = *drv_priv; + struct wd_alg_driver *driver = ctx_config->drv; + void *priv = ctx_config->drv_priv; + + if (!driver) + return; + + /* Prevent repeated uninitialization */ + if (!driver->init_state) + return;
driver->exit(priv); + driver->init_state = 0; /* Ctx config just need clear once */ wd_clear_ctx_config(config);
@@ -2048,8 +2095,38 @@ void wd_alg_uninit_driver(struct wd_ctx_config_internal *config,
if (priv) { free(priv); - *drv_priv = NULL; + ctx_config->drv_priv = NULL; + } +} + +int wd_alg_init_driver(struct wd_ctx_config_internal *config) +{ + __u32 i, j; + int ret; + + WD_ERR("debug: call function: %s!\n", __func__); + for (i = 0; i < config->ctx_num; i++) { + ret = wd_ctx_init_driver(config, &config->ctxs[i]); + if (ret) + goto init_err; } + + return 0; + +init_err: + for (j = 0; j < i; j++) + wd_ctx_uninit_driver(config, &config->ctxs[j]); + + return ret; +} + +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config) +{ + __u32 i; + + for (i = 0; i < config->ctx_num; i++) + wd_ctx_uninit_driver(config, &config->ctxs[i]); + }
void wd_dlclose_drv(void *dlh_list) @@ -2175,8 +2252,8 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params)
int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type) + char *alg, int task_type, enum wd_type type, + int max_op_type) { const char *env_name = wd_env_name[type]; const char *var_s; @@ -2190,9 +2267,9 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params,
/* Only hw driver support environment variable */ var_s = secure_getenv(env_name); - if (var_s && strlen(var_s) && driver->calc_type == UADK_ALG_HW) { + if (var_s && strlen(var_s) && task_type <= TASK_HW) { /* environment variable has the highest priority */ - ret = wd_env_set_ctx_nums(driver->alg_name, env_name, var_s, + ret = wd_env_set_ctx_nums(alg, env_name, var_s, ctx_params, max_op_type); if (ret) { WD_ERR("fail to init ctx nums from %s!\n", env_name); @@ -2203,33 +2280,28 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, /* environment variable is not set, try to use user_ctx_params first */ if (user_ctx_params) { copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); - ctx_params->cap = user_ctx_params->cap; - ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; - ctx_params->op_type_num = user_ctx_params->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { + if (user_ctx_params->op_type_num > (__u32)max_op_type) { WD_ERR("fail to check user op type numbers.\n"); numa_free_nodemask(ctx_params->bmp); return -WD_EINVAL; } + ctx_params->cap = user_ctx_params->cap; + ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; + ctx_params->op_type_num = user_ctx_params->op_type_num;
return 0; } - - /* user_ctx_params is also not set, use driver's defalut queue_num */ - numa_bitmask_setall(ctx_params->bmp); - for (i = 0; i < driver->op_type_num; i++) { - ctx_params->ctx_set_num[i].sync_ctx_num = driver->queue_num; - ctx_params->ctx_set_num[i].async_ctx_num = driver->queue_num; - } }
- ctx_params->op_type_num = driver->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { - WD_ERR("fail to check driver op type numbers.\n"); - numa_free_nodemask(ctx_params->bmp); - return -WD_EAGAIN; + /* user_ctx_params is also not set, use defalut queue_num max_op_type */ + numa_bitmask_setall(ctx_params->bmp); + for (i = 0; i < max_op_type; i++) { + ctx_params->ctx_set_num[i].sync_ctx_num = max_op_type; + ctx_params->ctx_set_num[i].async_ctx_num = max_op_type; }
+ ctx_params->op_type_num = max_op_type; + return 0; }
@@ -2356,48 +2428,60 @@ free_list: return NULL; }
-struct wd_alg_driver *wd_alg_drv_bind(int task_type, char *alg_name) +struct wd_alg_driver *wd_alg_drv_bind(__u8 ctx_type, char *alg_name) { - struct wd_alg_driver *set_driver = NULL; + struct wd_alg_driver *fb_drv; struct wd_alg_driver *drv;
- /* Get alg driver and dev name */ - switch (task_type) { - case TASK_INSTR: - drv = wd_request_drv(alg_name, true); + /* Get alg driver from ctx type and alg name */ + switch (ctx_type) { + case UADK_CTX_HW: + drv = wd_request_drv(alg_name, ALG_DRV_HW); if (!drv) { + WD_ERR("no HW %s driver support\n", alg_name); + return NULL; + } + + fb_drv = wd_request_drv(alg_name, ALG_DRV_CE_INS); + if (!fb_drv) { + drv->fallback = 0; WD_ERR("no soft %s driver support\n", alg_name); + } else { + drv->fallback = (handle_t)fb_drv; + WD_ERR("successful to get soft driver\n"); + } + + break; + case UADK_CTX_CE_INS: + drv = wd_request_drv(alg_name, ALG_DRV_CE_INS); + if (!drv) { + WD_ERR("no CE instr soft %s driver support\n", alg_name); return NULL; } - set_driver = drv; - set_driver->fallback = 0; + drv->fallback = 0; break; - case TASK_HW: - case TASK_MIX: - drv = wd_request_drv(alg_name, false); + case UADK_CTX_SVE_INS: + drv = wd_request_drv(alg_name, ALG_DRV_SVE_INS); if (!drv) { - WD_ERR("no HW %s driver support\n", alg_name); + WD_ERR("no SVE instr soft %s driver support\n", alg_name); return NULL; } - set_driver = drv; - set_driver->fallback = 0; - if (task_type == TASK_MIX) { - drv = wd_request_drv(alg_name, true); - if (!drv) { - set_driver->fallback = 0; - WD_ERR("no soft %s driver support\n", alg_name); - } else { - set_driver->fallback = (handle_t)drv; - WD_ERR("successful to get soft driver\n"); - } + drv->fallback = 0; + break; + case UADK_CTX_SOFT: + drv = wd_request_drv(alg_name, ALG_DRV_SOFT); + if (!drv) { + WD_ERR("no instr soft %s driver support\n", alg_name); + return NULL; } + drv->fallback = 0; break; default: - WD_ERR("task type error.\n"); - return NULL; + WD_ERR("ctx type error: %d.\n", ctx_type); + return WD_ERR_PTR(-WD_ENODEV); }
- return set_driver; + return drv; }
void wd_alg_drv_unbind(struct wd_alg_driver *drv) @@ -2413,19 +2497,56 @@ void wd_alg_drv_unbind(struct wd_alg_driver *drv) wd_release_drv(drv); }
-static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) +static __u32 wd_get_ctx_numbers(struct wd_init_attrs *attrs) { + struct wd_ctx_params *ctx_params = attrs->ctx_params; + int end = ctx_params->op_type_num; __u32 count = 0; int i;
for (i = 0; i < end; i++) { - count += ctx_params.ctx_set_num[i].sync_ctx_num; - count += ctx_params.ctx_set_num[i].async_ctx_num; + count += ctx_params->ctx_set_num[i].sync_ctx_num; + count += ctx_params->ctx_set_num[i].async_ctx_num; }
return count; }
+void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config) +{ + __u32 i; + + // wd_dlclose_drv after this + for (i = 0; i < ctx_config->ctx_num; i++) + wd_alg_drv_unbind(ctx_config->ctxs[i].drv); + +} + +int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config) +{ + __u32 i, j; + + // wd_dlopen_drv before this + WD_ERR("debug: call function: %s!\n", __func__); + for (i = 0; i < ctx_config->ctx_num; i++) { + ctx_config->ctxs[i].drv = wd_alg_drv_bind(ctx_config->ctxs[i].ctx_type, alg_name); + if (WD_IS_ERR(ctx_config->ctxs[i].drv)) { + continue; + } else if (!ctx_config->ctxs[i].drv) { + WD_ERR("failed to bind %s driver.\n", alg_name); + goto bind_err; + } + } + + return 0; + +bind_err: + for (j = 0; j < i; j++) { + wd_alg_drv_unbind(ctx_config->ctxs[j].drv); + } + return -WD_EINVAL; +} + struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) { struct uacce_dev_list *p, *node, *result = NULL; @@ -2520,6 +2641,7 @@ static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *l ctx_config->ctxs[i].ctx_mode = ((i - idx) < ctx_nums.sync_ctx_num) ? CTX_MODE_SYNC : CTX_MODE_ASYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_HW; }
return 0; @@ -2547,6 +2669,7 @@ static int wd_instance_sched_set(struct wd_sched *sched, struct wd_ctx_nums ctx_ sparams.type = op_type; sparams.mode = i; sparams.begin = idx + ctx_nums.sync_ctx_num * i; + sparams.ctx_prop = UADK_CTX_HW; end = idx - 1 + ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num * i; if (end < 0 || sparams.begin > (__u32)end) continue; @@ -2613,6 +2736,7 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs) __u32 ctx_set_num, op_type_num; int numa_cnt, ret;
+ WD_ERR("debug: call function: %s!\n", __func__); list = wd_get_accel_list(attrs->alg); if (!list) { WD_ERR("failed to get devices!\n"); @@ -2620,7 +2744,7 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs) }
op_type_num = ctx_params->op_type_num; - ctx_set_num = wd_get_ctx_numbers(*ctx_params, op_type_num); + ctx_set_num = wd_get_ctx_numbers(attrs); if (!ctx_set_num || !op_type_num) { WD_ERR("invalid: ctx_set_num is %u, op_type_num is %u!\n", ctx_set_num, op_type_num); @@ -2641,7 +2765,6 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs) }
wd_init_device_nodemask(used_list, used_bmp); - numa_cnt = numa_bitmask_weight(used_bmp); if (!numa_cnt) { ret = numa_cnt; @@ -2663,99 +2786,209 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs)
out_freeusedlist: wd_free_list_accels(used_list); + out_freelist: wd_free_list_accels(list);
return ret; }
-static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; + __u32 i;
- ctx_config->ctx_num = 1; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { - WD_ERR("failed to alloc ctxs!\n"); - return -WD_ENOMEM; + for (i = 0; i < ctx_config->ctx_num; i++) { + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_HW && ctx_config->ctxs[i].ctx) { + wd_release_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } }
- ctx_config->ctxs[0].ctx = (handle_t)calloc(1, sizeof(struct wd_ce_ctx)); - if (!ctx_config->ctxs[0].ctx) { + if (ctx_config->ctxs) { free(ctx_config->ctxs); - return -WD_ENOMEM; + ctx_config->ctxs = 0; } - - return WD_SUCCESS; }
-static void wd_alg_ce_ctx_uninit(struct wd_ctx_config *ctx_config) +static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs, bool mix_flag) { - __u32 i; + struct wd_ctx_config *ctx_config = attrs->ctx_config; + struct wd_soft_ctx *sfctx; + __u32 i, j, begin, end; + __u32 ctx_set_num; + size_t ctx_size; + __u32 mid_num; + + WD_ERR("debug: call function: %s!\n", __func__); + ctx_set_num = wd_get_ctx_numbers(attrs); + if (!mix_flag) { + ctx_config->ctx_num = ctx_set_num; + ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); + if (!ctx_config->ctxs) { + WD_ERR("failed to alloc ctxs!\n"); + return -WD_ENOMEM; + }
- for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - free((struct wd_ce_ctx *)ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; + begin = 0; + end = ctx_set_num; + mid_num = (begin + end) >> 1; + for (i = begin; i < end; i++) { + ctx_config->ctxs[i].op_type = 0; + if (i >= mid_num) + ctx_config->ctxs[i].ctx_mode = CTX_MODE_ASYNC; + else + ctx_config->ctxs[i].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_CE_INS; + sfctx = calloc(1, sizeof(struct wd_soft_ctx)); + if (!sfctx) { + WD_ERR("failed to alloc ctx!\n"); + goto ce_ctx_err; + } + ctx_config->ctxs[i].ctx = (handle_t)sfctx; + pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_SHARED); + pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_SHARED); + } + } else { + /* Need to supplement the soft computing queue space behind */ + ctx_config->ctx_num += ctx_set_num; + ctx_size = ctx_config->ctx_num * sizeof(struct wd_ctx); + ctx_config->ctxs = (struct wd_ctx *)realloc(ctx_config->ctxs, ctx_size); + if (!ctx_config->ctxs) { + WD_ERR("failed to realloc ce ctxs!\n"); + return -WD_ENOMEM; + } + + begin = ctx_config->ctx_num - ctx_set_num; + end = ctx_config->ctx_num; + mid_num = (begin + end) >> 1; + for (i = begin; i < end; i++) { + ctx_config->ctxs[i].op_type = 0; + if (i >= mid_num) + ctx_config->ctxs[i].ctx_mode = CTX_MODE_ASYNC; + else + ctx_config->ctxs[i].ctx_mode = CTX_MODE_SYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_CE_INS; + sfctx = calloc(1, sizeof(struct wd_soft_ctx)); + if (!sfctx) { + WD_ERR("failed to alloc ctx!\n"); + goto ce_ctx_err; + } + ctx_config->ctxs[i].ctx = (handle_t)sfctx; + pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_SHARED); + pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_SHARED); } }
- free(ctx_config->ctxs); + return WD_SUCCESS; + +ce_ctx_err: + for (j = i; j >= begin; j--) { + free((struct wd_soft_ctx *)ctx_config->ctxs[j].ctx); + ctx_config->ctxs[j].ctx = 0; + } + + return -WD_ENOMEM; }
-static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) +static void wd_alg_ce_ctx_uninit(struct wd_ctx_config *ctx_config) { __u32 i;
for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_CE_INS && ctx_config->ctxs[i].ctx) { + free((struct wd_soft_ctx *)ctx_config->ctxs[i].ctx); ctx_config->ctxs[i].ctx = 0; } }
- free(ctx_config->ctxs); + if (ctx_config->ctxs) { + free(ctx_config->ctxs); + ctx_config->ctxs = 0; + } }
-static int wd_alg_init_sve_ctx(struct wd_ctx_config *ctx_config) +static int wd_alg_sve_ctx_init(struct wd_init_attrs *attrs, bool mix_flag) { - struct wd_soft_ctx *ctx_sync, *ctx_async; - - ctx_config->ctx_num = WD_SOFT_CTX_NUM; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) - return -WD_ENOMEM; - - ctx_sync = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_sync) - goto free_ctxs; - - ctx_config->ctxs[WD_SOFT_SYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + struct wd_ctx_config *ctx_config = attrs->ctx_config; + __u32 i, j, begin, end; + __u32 ctx_set_num; + size_t ctx_size; + + ctx_set_num = wd_get_ctx_numbers(attrs); + if (!mix_flag) { + ctx_config->ctx_num = ctx_set_num; + ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); + if (!ctx_config->ctxs) + return -WD_ENOMEM;
- ctx_async = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_async) - goto free_ctx_sync; + begin = 0; + end = ctx_set_num; + for (i = begin; i < end; i++) { + ctx_config->ctxs[i].op_type = 0; + if (i % 2 == 0) + ctx_config->ctxs[i].ctx_mode = CTX_MODE_SYNC; + else + ctx_config->ctxs[i].ctx_mode = CTX_MODE_ASYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_SVE_INS; + ctx_config->ctxs[i].ctx = (handle_t)calloc(1, sizeof(struct wd_soft_ctx)); + if (!ctx_config->ctxs[i].ctx) { + WD_ERR("failed to alloc ctx!\n"); + goto sve_ctx_err; + } + } + } else { + /* Need to supplement the soft computing queue space behind */ + ctx_config->ctx_num += ctx_set_num; + ctx_size = ctx_config->ctx_num * sizeof(struct wd_ctx); + ctx_config->ctxs = (struct wd_ctx *)realloc(ctx_config->ctxs, ctx_size); + if (!ctx_config->ctxs) { + WD_ERR("failed to realloc sve ctxs!\n"); + return -WD_ENOMEM; + }
- ctx_config->ctxs[WD_SOFT_ASYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; + begin = ctx_config->ctx_num - ctx_set_num; + end = ctx_config->ctx_num; + for (i = begin; i < end; i++) { + ctx_config->ctxs[i].op_type = 0; + if (i % 2 == 0) + ctx_config->ctxs[i].ctx_mode = CTX_MODE_SYNC; + else + ctx_config->ctxs[i].ctx_mode = CTX_MODE_ASYNC; + ctx_config->ctxs[i].ctx_type = UADK_CTX_SVE_INS; + ctx_config->ctxs[i].ctx = (handle_t)calloc(1, sizeof(struct wd_soft_ctx)); + if (!ctx_config->ctxs[i].ctx) { + WD_ERR("failed to alloc ctx!\n"); + goto sve_ctx_err; + } + } + }
return 0;
-free_ctx_sync: - free(ctx_sync); -free_ctxs: - free(ctx_config->ctxs); +sve_ctx_err: + for (j = i; j >= begin; j--) { + free((struct wd_soft_ctx *)ctx_config->ctxs[j].ctx); + ctx_config->ctxs[j].ctx = 0; + } + return -WD_ENOMEM; }
-static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) +static void wd_alg_sve_ctx_uninit(struct wd_ctx_config *ctx_config) { - free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)(uintptr_t)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); - free(ctx_config->ctxs); + __u32 i; + + for (i = 0; i < ctx_config->ctx_num; i++) { + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_SVE_INS && ctx_config->ctxs[i].ctx) { + free((struct wd_soft_ctx *)ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + } + + if (ctx_config->ctxs) { + free(ctx_config->ctxs); + ctx_config->ctxs = 0; + } }
int wd_alg_attrs_init(struct wd_init_attrs *attrs) @@ -2766,117 +2999,214 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) struct wd_ctx_config *ctx_config = NULL; struct wd_sched *alg_sched = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; - int driver_type = UADK_ALG_HW; - char *alg = attrs->alg; - int ret = -WD_EINVAL; + char *alg_name = attrs->alg; + int task_type = attrs->task_type; + struct sched_params sparams; + struct wd_alg_driver *drv = NULL; + __u32 ctx_set_num, ctx_sync_num; + __u32 op_type_num; + int ret = 0;
if (!attrs->ctx_params) return -WD_EINVAL;
- if (attrs->driver) - driver_type = attrs->driver->calc_type; + ctx_config = calloc(1, sizeof(*ctx_config)); + if (!ctx_config) { + WD_ERR("fail to alloc ctx config\n"); + return -WD_ENOMEM; + } + attrs->ctx_config = ctx_config;
- switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; + /* Get op_type_num */ + op_type_num = attrs->ctx_params->op_type_num; + if (!op_type_num) + goto out_ctx_config;
- /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - goto out_ctx_config; - } + /* Use default sched_type to alloc scheduler */ + alg_sched = wd_sched_rr_alloc(sched_type, op_type_num, + numa_max_node() + 1, alg_poll_func); + if (!alg_sched) { + WD_ERR("fail to alloc scheduler\n"); + goto out_ctx_config; + } + attrs->sched = alg_sched; + memset(&sparams, 0x0, sizeof(struct sched_params));
- attrs->sched = alg_sched; + /* Initialize queues according to task type */ + switch (task_type) { + /* HW and fallback with soft driver */ + case TASK_HW: + wd_get_alg_type(alg_name, alg_type); + attrs->alg = alg_type;
- ret = wd_alg_ce_ctx_init(attrs); + ret = wd_alg_ctx_init(attrs); if (ret) { - WD_ERR("fail to init ce ctx\n"); + WD_ERR("fail to init HW ctx\n"); goto out_freesched; }
- ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; - break; - case UADK_ALG_SVE_INSTR: - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_SINGLE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - return -WD_EINVAL; - } - attrs->sched = alg_sched; - - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - goto out_freesched; - } - attrs->ctx_config = ctx_config; + /* HW and soft driver, soft ctx num same with HW ctx num */ + case TASK_MIX: + wd_get_alg_type(alg_name, alg_type); + attrs->alg = alg_type;
- ret = wd_alg_init_sve_ctx(ctx_config); + ret = wd_alg_ctx_init(attrs); if (ret) { - WD_ERR("fail to init sve ctx!\n"); + WD_ERR("fail to init HW ctx\n"); goto out_freesched; }
- ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) { - wd_alg_uninit_sve_ctx(ctx_config); - goto out_freesched; - } - break; - case UADK_ALG_HW: - wd_get_alg_type(alg, alg_type); - attrs->alg = alg_type; + WD_ERR("debug: call function: %s!\n", __func__); + drv = wd_request_drv(alg_name, ALG_DRV_SOFT); + if (drv == NULL) { + WD_ERR("fail to find soft driver.\n"); + break; + } else if (drv->calc_type == UADK_ALG_CE_INSTR) { + ret = wd_alg_ce_ctx_init(attrs, true); + if (ret) { + WD_ERR("fail to init ce ctx\n"); + goto out_hw_init; + }
- ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; + /* sync half and async half */ + ctx_set_num = wd_get_ctx_numbers(attrs); + ctx_sync_num = ctx_set_num >> 1; + sparams.begin = ctx_config->ctx_num - ctx_set_num; + sparams.end = ctx_config->ctx_num - ctx_sync_num - 1; + sparams.mode = CTX_MODE_SYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + sparams.begin = ctx_config->ctx_num - ctx_sync_num; + sparams.end = ctx_config->ctx_num - 1; + sparams.mode = CTX_MODE_ASYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + } else if (drv->calc_type == UADK_ALG_SVE_INSTR) { + ret = wd_alg_sve_ctx_init(attrs, true); + if (ret) { + WD_ERR("fail to init sve ctx\n"); + goto out_hw_init; + }
- alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - numa_max_node() + 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to instance scheduler\n"); - goto out_ctx_config; + ctx_set_num = wd_get_ctx_numbers(attrs); + ctx_sync_num = ctx_set_num >> 1; + sparams.begin = ctx_config->ctx_num - ctx_set_num; + sparams.end = ctx_config->ctx_num - ctx_sync_num - 1; + sparams.mode = CTX_MODE_SYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + sparams.begin = ctx_config->ctx_num - ctx_sync_num; + sparams.end = ctx_config->ctx_num - 1; + sparams.mode = CTX_MODE_ASYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } } - attrs->sched = alg_sched;
- ret = wd_alg_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ctx\n"); - goto out_freesched; - } + break; + /* Only pure soft queues */ + case TASK_INSTR: + drv = wd_request_drv(alg_name, ALG_DRV_SOFT); + if (drv == NULL) { + WD_ERR("fail to find soft driver.\n"); + goto out_ctx_init; + } else if (drv->calc_type == UADK_ALG_CE_INSTR) { + ret = wd_alg_ce_ctx_init(attrs, false); + if (ret) { + WD_ERR("fail to init ce ctx\n"); + return -WD_EINVAL; + }
- ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; + sparams.begin = 0; + sparams.end = (ctx_config->ctx_num >> 1) - 1; + sparams.mode = CTX_MODE_SYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + sparams.begin = ctx_config->ctx_num >> 1; + sparams.end = ctx_config->ctx_num - 1; + sparams.mode = CTX_MODE_ASYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_CE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + } else if (drv->calc_type == UADK_ALG_SVE_INSTR) { + ret = wd_alg_sve_ctx_init(attrs, true); + if (ret) { + WD_ERR("fail to init sve ctx\n"); + return -WD_EINVAL; + } + + sparams.begin = 0; + sparams.end = (ctx_config->ctx_num >> 1) - 1; + sparams.mode = CTX_MODE_SYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + sparams.begin = ctx_config->ctx_num >> 1; + sparams.end = ctx_config->ctx_num - 1; + sparams.mode = CTX_MODE_ASYNC; + sparams.numa_id = 0; + sparams.ctx_prop = UADK_CTX_SVE_INS; + ret = wd_sched_rr_instance(alg_sched, &sparams); + if (ret) { + WD_ERR("fail to instance scheduler.\n"); + goto out_ctx_init; + } + } break; default: - WD_ERR("driver type error: %d\n", driver_type); + WD_ERR("driver type error: %d\n", drv->calc_type); return -WD_EINVAL; }
+ ctx_config->cap = attrs->ctx_params->cap; + ret = alg_init_func(ctx_config, alg_sched); + if (ret) + goto out_hw_init; + + WD_ERR("---->ctx nums: %u\n", ctx_config->ctx_num); + return 0;
-out_pre_init: - if (driver_type == UADK_ALG_CE_INSTR || driver_type == UADK_ALG_SOFT) +out_hw_init: + wd_alg_ctx_uninit(ctx_config); +out_ctx_init: + if (drv->calc_type == UADK_ALG_CE_INSTR) wd_alg_ce_ctx_uninit(ctx_config); - else - wd_alg_ctx_uninit(ctx_config); + else if (drv->calc_type == UADK_ALG_SVE_INSTR) + wd_alg_sve_ctx_uninit(ctx_config); out_freesched: wd_sched_rr_release(alg_sched); out_ctx_config: @@ -2885,32 +3215,112 @@ out_ctx_config: return ret; }
+static void wd_alg_ctxs_uninit(struct wd_ctx_config *ctx_config) +{ + __u32 i; + + for (i = 0; i < ctx_config->ctx_num; i++) { + if (!ctx_config->ctxs[i].ctx) + continue; + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_HW) { + wd_release_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_CE_INS) { + free((struct wd_ctx *)ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + + if (ctx_config->ctxs[i].ctx_type == UADK_CTX_SVE_INS) { + free((struct wd_ctx *)ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + } + + if (ctx_config->ctxs) { + free(ctx_config->ctxs); + ctx_config->ctxs = 0; + } +} + void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) { struct wd_ctx_config *ctx_config = attrs->ctx_config; struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type;
if (!ctx_config) { wd_sched_rr_release(alg_sched); return; }
- switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - wd_alg_ce_ctx_uninit(ctx_config); - break; - case UADK_ALG_SVE_INSTR: - wd_alg_uninit_sve_ctx(ctx_config); - break; - case UADK_ALG_HW: - wd_alg_ctx_uninit(ctx_config); - break; - default: - break; - } + wd_alg_ctxs_uninit(ctx_config);
free(ctx_config); wd_sched_rr_release(alg_sched); } + +int wd_queue_is_busy(struct wd_soft_ctx *sctx) +{ + /* The queue is not used */ + if (sctx->run_num >= MAX_SOFT_QUEUE_LENGTH - 1) + return -WD_EBUSY; + + return 0; +} + +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +{ + struct wd_soft_sqe *sqe = NULL; + + pthread_spin_lock(&sctx->slock); + sqe = &sctx->qfifo[sctx->head]; + if (!sqe->used && !sqe->complete) { // find the next not used sqe + sctx->head++; + if (sctx->head == MAX_SOFT_QUEUE_LENGTH) + sctx->head = 0; + + __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + sqe->used = 1; + sqe->complete = 1; + sqe->id = tag_id; + sqe->result = 0; + pthread_spin_unlock(&sctx->slock); + } else { + pthread_spin_unlock(&sctx->slock); + return -WD_EBUSY; + } + + return 0; +} + +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) +{ + struct wd_soft_sqe *sqe = NULL; + + /* The queue is not used */ + if (sctx->run_num < 1) + return -WD_EAGAIN; + + pthread_spin_lock(&sctx->rlock); + sqe = &sctx->qfifo[sctx->tail]; + if (sqe->used && sqe->complete) { // find a used sqe + sctx->tail++; + if (sctx->tail == MAX_SOFT_QUEUE_LENGTH) + sctx->tail = 0; + + *tag_id = sqe->id; + *result = sqe->result; + sqe->used = 0x0; + sqe->complete = 0x0; + __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->rlock); + } else { + pthread_spin_unlock(&sctx->rlock); + return -WD_EAGAIN; + } + + return 0; +} +