Support sync: mempcy, user pointer and sgl case Support async: memcpy, user pointer
For flat memory: If user call wd_xxx_setup_blkpool, will use user pointer mode. uadk directly use pointer for app, assume it is continuous memory and translate va to pa when configure register
Otherwise, alloc_sess will setup blkpool and use memcpy mode. wd_comp alloc continuous memory for hardware and memcpy from src pointer and memcpy results to dst pointer
For sgl memory: App has to call wd_xxx_setup_blkpool. The wd_datalist.data has to use continuous memory
Signed-off-by: Zhangfei Gao zhangfei.gao@linaro.org --- drv/hisi_comp.c | 102 ++++++++++++++++++++++++++++---------- drv/hisi_qm_udrv.c | 14 ++++-- drv/hisi_qm_udrv.h | 3 +- drv/hisi_sec.c | 8 +-- include/drv/wd_comp_drv.h | 7 +++ wd_comp.c | 98 ++++++++++++++++++++++++++++++++++++ 6 files changed, 197 insertions(+), 35 deletions(-)
diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index cd558a8..7de055a 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -431,7 +431,14 @@ static int fill_buf_deflate_generic(struct hisi_zip_sqe *sqe, if (msg->ctx_buf) ctx_buf = msg->ctx_buf + RSV_OFFSET;
- fill_buf_addr_deflate(sqe, src, dst, ctx_buf); + if (msg->blkpool) { + fill_buf_addr_deflate(sqe, + wd_blkpool_phy(msg->blkpool, src), + wd_blkpool_phy(msg->blkpool, dst), + wd_blkpool_phy(msg->blkpool, ctx_buf)); + } else { + fill_buf_addr_deflate(sqe, src, dst, ctx_buf); + }
return 0; } @@ -464,32 +471,45 @@ static void fill_buf_type_sgl(struct hisi_zip_sqe *sqe) }
static int fill_buf_addr_deflate_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, + struct wd_comp_msg *msg, struct wd_datalist *list_src, struct wd_datalist *list_dst) { void *hw_sgl_in, *hw_sgl_out; handle_t h_sgl_pool;
- h_sgl_pool = hisi_qm_get_sglpool(h_qp); + if (msg->h_sgl_pool) + h_sgl_pool = msg->h_sgl_pool; + else + h_sgl_pool = hisi_qm_get_sglpool(h_qp); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool!\n"); return -WD_EINVAL; }
- hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, list_src); + hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, list_src, msg->blkpool); if (unlikely(!hw_sgl_in)) { WD_ERR("failed to get hw sgl in!\n"); - return -WD_ENOMEM; + return -WD_EBUSY; }
- hw_sgl_out = hisi_qm_get_hw_sgl(h_sgl_pool, list_dst); + hw_sgl_out = hisi_qm_get_hw_sgl(h_sgl_pool, list_dst, msg->blkpool); if (unlikely(!hw_sgl_out)) { WD_ERR("failed to get hw sgl out!\n"); hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); - return -WD_ENOMEM; + return -WD_EBUSY; }
- fill_buf_addr_deflate(sqe, hw_sgl_in, hw_sgl_out, NULL); + if (msg->h_sgl_pool) { + fill_buf_addr_deflate(sqe, + wd_blkpool_phy(msg->blkpool, hw_sgl_in), + wd_blkpool_phy(msg->blkpool, hw_sgl_out), + NULL); + msg->hw_sgl_in = hw_sgl_in; + msg->hw_sgl_out = hw_sgl_out; + } else { + fill_buf_addr_deflate(sqe, hw_sgl_in, hw_sgl_out, NULL); + }
return 0; } @@ -543,7 +563,7 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe,
fill_buf_type_sgl(sqe);
- ret = fill_buf_addr_deflate_sgl(h_qp, sqe, list_src, list_dst); + ret = fill_buf_addr_deflate_sgl(h_qp, sqe, msg, list_src, list_dst); if (unlikely(ret)) return ret;
@@ -738,34 +758,48 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe,
fill_buf_size_lz77_zstd(sqe, in_size, lits_size, out_size - lits_size);
- h_sgl_pool = hisi_qm_get_sglpool(h_qp); + if (msg->h_sgl_pool) + h_sgl_pool = msg->h_sgl_pool; + else + h_sgl_pool = hisi_qm_get_sglpool(h_qp); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool!\n"); return -WD_EINVAL; }
- hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, req->list_src); + hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, req->list_src, msg->blkpool); if (unlikely(!hw_sgl_in)) { WD_ERR("failed to get hw sgl in!\n"); return -WD_ENOMEM; }
- hw_sgl_out_lit = hisi_qm_get_hw_sgl(h_sgl_pool, req->list_dst); + hw_sgl_out_lit = hisi_qm_get_hw_sgl(h_sgl_pool, req->list_dst, msg->blkpool); if (unlikely(!hw_sgl_out_lit)) { WD_ERR("failed to get hw sgl out for literals!\n"); ret = -WD_ENOMEM; goto err_free_sgl_in; }
- hw_sgl_out_seq = hisi_qm_get_hw_sgl(h_sgl_pool, seq_start); + hw_sgl_out_seq = hisi_qm_get_hw_sgl(h_sgl_pool, seq_start, msg->blkpool); if (unlikely(!hw_sgl_out_seq)) { WD_ERR("failed to get hw sgl out for sequences!\n"); ret = -WD_ENOMEM; goto err_free_sgl_out_lit; }
- fill_buf_addr_lz77_zstd(sqe, hw_sgl_in, hw_sgl_out_lit, + if (msg->h_sgl_pool) { + fill_buf_addr_lz77_zstd(sqe, + wd_blkpool_phy(msg->blkpool, hw_sgl_in), + wd_blkpool_phy(msg->blkpool, hw_sgl_out_lit), + wd_blkpool_phy(msg->blkpool, hw_sgl_out_seq), + NULL); + msg->hw_sgl_in = hw_sgl_in; + msg->hw_sgl_out = hw_sgl_out_lit; + msg->hw_sgl_out_seq = hw_sgl_out_seq; + } else { + fill_buf_addr_lz77_zstd(sqe, hw_sgl_in, hw_sgl_out_lit, hw_sgl_out_seq, NULL); + }
return 0;
@@ -1116,27 +1150,41 @@ static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, }
static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, + struct wd_comp_msg *msg, enum wd_comp_alg_type alg_type) { void *hw_sgl_in, *hw_sgl_out; handle_t h_sgl_pool;
- h_sgl_pool = hisi_qm_get_sglpool(h_qp); - if (unlikely(!h_sgl_pool)) { - WD_ERR("failed to get sglpool to free hw sgl!\n"); - return; - } + if (msg->h_sgl_pool) { + h_sgl_pool = msg->h_sgl_pool; + if (unlikely(!h_sgl_pool)) { + WD_ERR("failed to get sglpool to free hw sgl!\n"); + return; + } + hisi_qm_put_hw_sgl(h_sgl_pool, msg->hw_sgl_in); + hisi_qm_put_hw_sgl(h_sgl_pool, msg->hw_sgl_out); + if (alg_type == WD_LZ77_ZSTD) + hisi_qm_put_hw_sgl(h_sgl_pool, msg->hw_sgl_out_seq); + } else {
- hw_sgl_in = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); + h_sgl_pool = hisi_qm_get_sglpool(h_qp); + if (unlikely(!h_sgl_pool)) { + WD_ERR("failed to get sglpool to free hw sgl!\n"); + return; + }
- hw_sgl_out = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); + hw_sgl_in = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); + hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in);
- if (alg_type == WD_LZ77_ZSTD) { - hw_sgl_out = VA_ADDR(sqe->literals_addr_h, - sqe->literals_addr_l); + hw_sgl_out = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); + + if (alg_type == WD_LZ77_ZSTD) { + hw_sgl_out = VA_ADDR(sqe->literals_addr_h, + sqe->literals_addr_l); + hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); + } } }
@@ -1163,7 +1211,7 @@ static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *com ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (unlikely(ret < 0)) { if (msg->req.data_fmt == WD_SGL_BUF) - free_hw_sgl(h_qp, &sqe, msg->alg_type); + free_hw_sgl(h_qp, &sqe, msg, msg->alg_type); if (ret != -WD_EBUSY) WD_ERR("failed to send to hardware, ret = %d!\n", ret);
@@ -1304,7 +1352,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, recv_msg->alg_type = alg_type;
if (buf_type == WD_SGL_BUF) - free_hw_sgl((handle_t)qp, sqe, alg_type); + free_hw_sgl((handle_t)qp, sqe, recv_msg, alg_type);
if (unlikely(recv_msg->req.status == WD_IN_EPARA)) dump_zip_msg(recv_msg); diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 304764e..78f6583 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -9,6 +9,7 @@
#include "hisi_qm_udrv.h" #include "wd_util.h" +#include "wd_bmm.h"
#define QM_DBELL_CMD_SQ 0 #define QM_DBELL_CMD_CQ 1 @@ -842,7 +843,8 @@ static void hisi_qm_dump_sgl(void *sgl) } }
-void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) +void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl, + void *blkpool) { struct hisi_sgl_pool *pool = (struct hisi_sgl_pool *)sgl_pool; struct wd_datalist *tmp = sgl; @@ -872,7 +874,10 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) goto err_out; }
- cur->sge_entries[i].buff = (uintptr_t)tmp->data; + if (blkpool) + cur->sge_entries[i].buff = (uintptr_t)wd_blkpool_phy(blkpool, tmp->data); + else + cur->sge_entries[i].buff = (uintptr_t)tmp->data; cur->sge_entries[i].len = tmp->len; cur->entry_sum_in_sgl++; cur->entry_size_in_sgl += tmp->len; @@ -890,7 +895,10 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) WD_ERR("invalid: the sgl pool is not enough!\n"); goto err_out; } - cur->next_dma = (uintptr_t)next; + if (blkpool) + cur->next_dma = (uintptr_t)wd_blkpool_phy(blkpool, next); + else + cur->next_dma = (uintptr_t)next; cur = next; head->entry_sum_in_chain += pool->sge_num; /* In the new sgl chain, the subscript must be reset */ diff --git a/drv/hisi_qm_udrv.h b/drv/hisi_qm_udrv.h index b02e8e7..ddb666e 100644 --- a/drv/hisi_qm_udrv.h +++ b/drv/hisi_qm_udrv.h @@ -162,7 +162,8 @@ void hisi_qm_destroy_sglpool(handle_t sgl_pool); * * Return the hw sgl addr which can fill into the sqe. */ -void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl); +void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl, + void *blkpool);
/** * hisi_qm_put_hw_sgl - Reback the hw sgl to the sgl pool. diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 747d3a8..a305985 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1050,7 +1050,7 @@ static int hisi_sec_fill_sgl(handle_t h_qp, __u8 **in, __u8 **out, return -WD_EINVAL; }
- hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in)); + hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in), NULL); if (!hw_sgl_in) { WD_ERR("failed to get sgl in for hw_v2!\n"); return -WD_EINVAL; @@ -1060,7 +1060,7 @@ static int hisi_sec_fill_sgl(handle_t h_qp, __u8 **in, __u8 **out, hw_sgl_out = *out; } else { hw_sgl_out = hisi_qm_get_hw_sgl(h_sgl_pool, - (struct wd_datalist *)(*out)); + (struct wd_datalist *)(*out), NULL); if (!hw_sgl_out) { WD_ERR("failed to get hw sgl out for hw_v2!\n"); hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); @@ -1090,7 +1090,7 @@ static int hisi_sec_fill_sgl_v3(handle_t h_qp, __u8 **in, __u8 **out, return -WD_EINVAL; }
- hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in)); + hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in), NULL); if (!hw_sgl_in) { WD_ERR("failed to get sgl in for hw_v3!\n"); return -WD_EINVAL; @@ -1101,7 +1101,7 @@ static int hisi_sec_fill_sgl_v3(handle_t h_qp, __u8 **in, __u8 **out, sqe->bd_param |= SEC_PBUFF_MODE_MASK_V3; } else { hw_sgl_out = hisi_qm_get_hw_sgl(h_sgl_pool, - (struct wd_datalist *)(*out)); + (struct wd_datalist *)(*out), NULL); if (!hw_sgl_out) { WD_ERR("failed to get hw sgl out for hw_v3!\n"); hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d..1b8273c 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -53,6 +53,13 @@ struct wd_comp_msg { __u32 checksum; /* Request identifier */ __u32 tag; + void *blkpool; + void *src; + void *dst; + handle_t h_sgl_pool; + void *hw_sgl_in; + void *hw_sgl_out; + void *hw_sgl_out_seq; };
struct wd_comp_msg *wd_comp_get_msg(__u32 idx, __u32 tag); diff --git a/wd_comp.c b/wd_comp.c index 692abf4..f26c5da 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -39,6 +39,10 @@ struct wd_comp_sess { __u32 checksum; __u8 *ctx_buf; void *sched_key; + void *blkpool; + __u8 *blkpool_ctxbuf; + handle_t h_sgl_pool; + __u8 blkpool_mode; };
struct wd_comp_setting { @@ -361,6 +365,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) struct wd_comp_req *req; __u64 recv_count = 0; __u32 tmp = expt; + void *blkpool = NULL; int ret;
if (unlikely(!count || !expt)) { @@ -374,6 +379,10 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) if (unlikely(ret)) return ret;
+ ctx = config->ctxs; + if (ctx->blkpool && ctx->blkpool_mode == BLKPOOL_MODE_MEMCPY) + blkpool = ctx->blkpool; + ctx = config->ctxs + idx;
do { @@ -394,6 +403,11 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) }
req = &msg->req; + if (blkpool) { + memcpy(msg->dst, req->dst, msg->produced); + wd_blkpool_free(blkpool, msg->req.src); + wd_blkpool_free(blkpool, msg->req.dst); + } req->src_len = msg->in_cons; req->dst_len = msg->produced; req->cb(req, req->cb_param); @@ -436,6 +450,8 @@ static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup)
handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { + struct wd_ctx_config_internal *config = &wd_comp_setting.config; + struct wd_ctx_internal *ctx = config->ctxs; struct wd_comp_sess *sess; int ret;
@@ -467,6 +483,28 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) goto sched_err; }
+ if (ctx->blkpool) { + sess->blkpool = ctx->blkpool; + sess->h_sgl_pool = ctx->h_sgl_pool; + sess->blkpool_mode = ctx->blkpool_mode; + + if (ctx->blkpool_mode == BLKPOOL_MODE_MEMCPY) { + struct wd_blkpool_setup blksetup; + + memset(&blksetup, 0, sizeof(blksetup)); + blksetup.block_size = HW_CTX_SIZE; + blksetup.block_num = DEFAULT_BLOCK_NM; + blksetup.align_size = DEFAULT_ALIGN_SIZE; + ret = wd_blkpool_setup(sess->blkpool, &blksetup); + if (ret) + goto sched_err; + } + + sess->blkpool_ctxbuf = wd_blkpool_alloc(sess->blkpool, HW_CTX_SIZE); + if (!sess->blkpool_ctxbuf) + goto sched_err; + } + return (handle_t)sess;
sched_err: @@ -486,6 +524,9 @@ void wd_comp_free_sess(handle_t h_sess) if (sess->ctx_buf) free(sess->ctx_buf);
+ if (sess->blkpool_ctxbuf) + wd_blkpool_free(sess->blkpool, sess->blkpool_ctxbuf); + if (sess->sched_key) free(sess->sched_key);
@@ -506,6 +547,9 @@ int wd_comp_reset_sess(handle_t h_sess) if (sess->ctx_buf) memset(sess->ctx_buf, 0, HW_CTX_SIZE);
+ if (sess->blkpool_ctxbuf) + memset(sess->blkpool_ctxbuf, 0, HW_CTX_SIZE); + return 0; }
@@ -630,6 +674,29 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx;
+ if (sess->blkpool) { + msg->ctx_buf = sess->blkpool_ctxbuf; + msg->blkpool = sess->blkpool; + msg->h_sgl_pool = sess->h_sgl_pool; + + if (sess->blkpool_mode == BLKPOOL_MODE_MEMCPY) { + void *src = wd_blkpool_alloc(sess->blkpool, req->src_len); + void *dst = wd_blkpool_alloc(sess->blkpool, req->dst_len); + + if (!src || !dst) + return -ENOMEM; + + /* save */ + msg->src = msg->req.src; + msg->dst = msg->req.dst; + /* replace */ + msg->req.src = src; + msg->req.dst = dst; + + memcpy(msg->req.src, msg->src, req->src_len); + } + } + msg_handle.send = wd_comp_setting.driver->send; msg_handle.recv = wd_comp_setting.driver->recv;
@@ -638,6 +705,14 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, msg, NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock);
+ if (sess->blkpool) { + if (sess->blkpool_mode == BLKPOOL_MODE_MEMCPY) { + memcpy(msg->dst, req->dst, msg->produced); + wd_blkpool_free(sess->blkpool, msg->req.src); + wd_blkpool_free(sess->blkpool, msg->req.dst); + } + } + return ret; }
@@ -893,6 +968,29 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS;
+ if (sess->blkpool) { + /* async mode has no ctx_buf */ + msg->blkpool = sess->blkpool; + msg->h_sgl_pool = sess->h_sgl_pool; + + if (sess->blkpool_mode == BLKPOOL_MODE_MEMCPY) { + void *src = wd_blkpool_alloc(sess->blkpool, req->src_len); + void *dst = wd_blkpool_alloc(sess->blkpool, req->dst_len); + + if (!src || !dst) + return -ENOMEM; + + /* save */ + msg->src = msg->req.src; + msg->dst = msg->req.dst; + /* replace */ + msg->req.src = src; + msg->req.dst = dst; + + memcpy(msg->req.src, msg->src, req->src_len); + } + } + ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); if (unlikely(ret < 0)) { WD_ERR("wd comp send error, ret = %d!\n", ret);