[Git-lng-odp] [COMMIT] branch master updated. v1.17.0.0_tigermoth_rc1-130-ge98c15a9

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".

The branch, master has been updated via e98c15a9d2ee9d1dd21e3d6c3d9829581d8328c0 (commit) via 1f4a5f79557186df79c2091dcc73b6e783c95f74 (commit) via 6c4ecdb73aa869e4733b9b323d9dc4c625a63e58 (commit) from 6be7adab1a16a62e871bc4ed4f3c926645b2ecd0 (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit e98c15a9d2ee9d1dd21e3d6c3d9829581d8328c0 Author: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Date: Thu Dec 14 04:22:12 2017 +0300

linux-gen: crypto: significant speedup of all operations

Per idea of Janne Peltonen, do not allocate/free crypto contexts for each operation, providing significant speed increase. Each thread on startup allocates hmac+cipher contexts pair for each crypto session. Then they are initialized on demand, when session is first executed using this thread.

Signed-off-by: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Reviewed-by: Janne Peltonen janne.peltonen@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 54f95b8b..4fd72144 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -110,6 +110,8 @@ int odp_queue_term_global(void);

int odp_crypto_init_global(void); int odp_crypto_term_global(void); +int _odp_crypto_init_local(void); +int _odp_crypto_term_local(void);

int odp_timer_init_global(const odp_init_t *params); int odp_timer_term_global(void); diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 3b110c19..8ae484fd 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -9,7 +9,6 @@ #include <odp_posix_extensions.h> #include <odp/api/crypto.h> #include <odp_internal.h> -#include <odp/api/atomic.h> #include <odp/api/spinlock.h> #include <odp/api/sync.h> #include <odp/api/debug.h> @@ -98,6 +97,7 @@ typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session); +typedef void (*crypto_init_func_t)(odp_crypto_generic_session_t *session);

/** * Per crypto session data structure @@ -117,6 +117,7 @@ struct odp_crypto_generic_session_t {

const EVP_CIPHER *evp_cipher; crypto_func_t func; + crypto_init_func_t init; } cipher;

struct { @@ -128,7 +129,10 @@ struct odp_crypto_generic_session_t { const EVP_CIPHER *evp_cipher; }; crypto_func_t func; + crypto_init_func_t init; } auth; + + unsigned idx; };

typedef struct odp_crypto_global_s odp_crypto_global_t; @@ -137,15 +141,42 @@ struct odp_crypto_global_s { odp_spinlock_t lock; odp_crypto_generic_session_t *free; odp_crypto_generic_session_t sessions[MAX_SESSIONS]; + + /* These flags are cleared at alloc_session() */ + uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; + odp_ticketlock_t openssl_lock[0]; };

static odp_crypto_global_t *global;

+typedef struct crypto_local_t { + HMAC_CTX *hmac_ctx[MAX_SESSIONS]; + EVP_CIPHER_CTX *cipher_ctx[MAX_SESSIONS]; + EVP_CIPHER_CTX *mac_cipher_ctx[MAX_SESSIONS]; + uint8_t *ctx_valid; +} crypto_local_t; + +static __thread crypto_local_t local; + +typedef enum { KIND_HMAC, KIND_CIPHER, KIND_MAC_CIPHER } crypto_kind_t; + +static inline void crypto_init(odp_crypto_generic_session_t *session) +{ + if (local.ctx_valid[session->idx]) + return; + + session->cipher.init(session); + session->auth.init(session); + + local.ctx_valid[session->idx] = 1; +} + static odp_crypto_generic_session_t *alloc_session(void) { odp_crypto_generic_session_t *session = NULL; + unsigned i;

odp_spinlock_lock(&global->lock); session = global->free; @@ -155,6 +186,11 @@ odp_crypto_generic_session_t *alloc_session(void) } odp_spinlock_unlock(&global->lock);

+ session->idx = session - global->sessions; + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) + global->ctx_valid[i][session->idx] = 0; + return session; }

@@ -175,24 +211,57 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, return ODP_CRYPTO_ALG_ERR_NONE; }

-static -void packet_hmac_calculate(HMAC_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) +static void +null_crypto_init_routine(odp_crypto_generic_session_t *session ODP_UNUSED) { - uint32_t offset = param->auth_range.offset; - uint32_t len = param->auth_range.length; + return; +}

- ODP_ASSERT(offset + len <= odp_packet_len(pkt)); +/* Mimic new OpenSSL 1.1.y API */ +#if OPENSSL_VERSION_NUMBER < 0x10100000L +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = malloc(sizeof(*ctx)); + + HMAC_CTX_init(ctx); + return ctx; +} + +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + HMAC_CTX_cleanup(ctx); + free(ctx); +} +#endif + +static void +auth_init(odp_crypto_generic_session_t *session) +{ + HMAC_CTX *ctx = local.hmac_ctx[session->idx];

HMAC_Init_ex(ctx, session->auth.key, session->auth.key_length, session->auth.evp_md, NULL); +} + +static +void packet_hmac(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) +{ + HMAC_CTX *ctx = local.hmac_ctx[session->idx]; + uint32_t offset = param->auth_range.offset; + uint32_t len = param->auth_range.length;

+ ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + + /* Reinitialize HMAC calculation without resetting the key */ + HMAC_Init_ex(ctx, NULL, 0, NULL, NULL); + + /* Hash it */ while (len > 0) { uint32_t seglen = 0; /* GCC */ void *mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); @@ -206,36 +275,6 @@ void packet_hmac_calculate(HMAC_CTX *ctx, HMAC_Final(ctx, hash, NULL); }

-#if OPENSSL_VERSION_NUMBER < 0x10100000L -static -void packet_hmac(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX ctx; - - /* Hash it */ - HMAC_CTX_init(&ctx); - packet_hmac_calculate(&ctx, pkt, param, session, hash); - HMAC_CTX_cleanup(&ctx); -} -#else -static -void packet_hmac(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX *ctx; - - /* Hash it */ - ctx = HMAC_CTX_new(); - packet_hmac_calculate(ctx, pkt, param, session, hash); - HMAC_CTX_free(ctx); -} -#endif - static odp_crypto_alg_err_t auth_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, @@ -425,12 +464,22 @@ int internal_decrypt(EVP_CIPHER_CTX *ctx, return ret; }

+static void +cipher_encrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; void *iv_ptr; int ret;

@@ -441,27 +490,30 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

ret = internal_encrypt(ctx, pkt, param);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; }

+static void +cipher_decrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; void *iv_ptr; int ret;

@@ -472,17 +524,10 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

ret = internal_decrypt(ctx, pkt, param);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } @@ -506,20 +551,35 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, session->p.cipher_key.length);

/* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->cipher.func = cipher_encrypt; - else + session->cipher.init = cipher_encrypt_init; + } else { session->cipher.func = cipher_decrypt; + session->cipher.init = cipher_decrypt_init; + }

return 0; }

+static void +aes_gcm_encrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; void *iv_ptr; @@ -534,14 +594,7 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

/* Authenticate header data (if any) without encrypting them */ if (aad_len > 0) @@ -555,18 +608,28 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, odp_packet_copy_from_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; }

+static void +aes_gcm_decrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; int dummy_len = 0; @@ -581,14 +644,7 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -602,8 +658,6 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,

ret = internal_decrypt(ctx, pkt, param);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : ODP_CRYPTO_ALG_ERR_NONE; } @@ -622,20 +676,35 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session, session->cipher.evp_cipher = cipher;

/* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->cipher.func = aes_gcm_encrypt; - else + session->cipher.init = aes_gcm_encrypt_init; + } else { session->cipher.func = aes_gcm_decrypt; + session->cipher.init = aes_gcm_decrypt_init; + }

return 0; }

+static void +aes_gmac_gen_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, + session->auth.key, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; @@ -647,14 +716,7 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, - session->auth.key, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

ret = internal_aad(ctx, pkt, param);

@@ -663,18 +725,28 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, odp_packet_copy_from_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; }

+static void +aes_gmac_check_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL, + session->auth.key, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; @@ -686,14 +758,7 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID;

- /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL, - session->auth.key, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0);

odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -704,8 +769,6 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt,

ret = internal_aad(ctx, pkt, param);

- EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : ODP_CRYPTO_ALG_ERR_NONE; } @@ -724,10 +787,13 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session, session->auth.evp_cipher = cipher;

/* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->auth.func = aes_gmac_gen; - else + session->auth.init = aes_gmac_gen_init; + } else { session->auth.func = aes_gmac_check; + session->auth.init = aes_gmac_check_init; + }

return 0; } @@ -741,6 +807,7 @@ static int process_auth_param(odp_crypto_generic_session_t *session, session->auth.func = auth_gen; else session->auth.func = auth_check; + session->auth.init = auth_init;

session->auth.evp_md = evp_md;

@@ -923,6 +990,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; + session->cipher.init = null_crypto_init_routine; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -998,6 +1066,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; + session->auth.init = null_crypto_init_routine; rc = 0; break; #if ODP_DEPRECATED_API @@ -1037,6 +1106,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, * cipher at the same time */ if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || aes_gcm) { session->auth.func = null_crypto_routine; + session->auth.init = null_crypto_init_routine; rc = 0; } else { rc = -1; @@ -1216,6 +1286,49 @@ int odp_crypto_term_global(void) return rc; }

+int _odp_crypto_init_local(void) +{ + unsigned i; + int id; + + memset(&local, 0, sizeof(local)); + + for (i = 0; i < MAX_SESSIONS; i++) { + local.hmac_ctx[i] = HMAC_CTX_new(); + local.cipher_ctx[i] = EVP_CIPHER_CTX_new(); + local.mac_cipher_ctx[i] = EVP_CIPHER_CTX_new(); + + if (local.hmac_ctx[i] == NULL || + local.cipher_ctx[i] == NULL || + local.mac_cipher_ctx[i] == NULL) { + _odp_crypto_term_local(); + return -1; + } + } + + id = odp_thread_id(); + local.ctx_valid = global->ctx_valid[id]; + /* No need to clear flags here, alloc_session did the job for us */ + + return 0; +} + +int _odp_crypto_term_local(void) +{ + unsigned i; + + for (i = 0; i < MAX_SESSIONS; i++) { + if (local.hmac_ctx[i] != NULL) + HMAC_CTX_free(local.hmac_ctx[i]); + if (local.cipher_ctx[i] != NULL) + EVP_CIPHER_CTX_free(local.cipher_ctx[i]); + if (local.mac_cipher_ctx[i] != NULL) + EVP_CIPHER_CTX_free(local.mac_cipher_ctx[i]); + } + + return 0; +} + odp_random_kind_t odp_random_max_kind(void) { return ODP_RANDOM_CRYPTO; @@ -1385,6 +1498,8 @@ int odp_crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; }

+ crypto_init(session); + /* Invoke the functions */ if (session->do_cipher_first) { rc_cipher = session->cipher.func(out_pkt, param, session); diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index bff8303e..bbcbd8dd 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -327,6 +327,12 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) } stage = PKTIO_INIT;

+ if (_odp_crypto_init_local()) { + ODP_ERR("ODP crypto local init failed.\n"); + goto init_fail; + } + stage = CRYPTO_INIT; + if (odp_pool_init_local()) { ODP_ERR("ODP pool local init failed.\n"); goto init_fail; @@ -379,6 +385,13 @@ int _odp_term_local(enum init_stage stage) } /* Fall through */

+ case CRYPTO_INIT: + if (_odp_crypto_term_local()) { + ODP_ERR("ODP crypto local term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (odp_pool_term_local()) { ODP_ERR("ODP buffer pool local term failed.\n");

commit 1f4a5f79557186df79c2091dcc73b6e783c95f74 Author: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Date: Thu Dec 14 12:26:21 2017 +0300

linux-gen: crypto: move session type to odp_crypto module

There is no point in having odp_crypto_generic_session_t definition in global include file. Move it to odp_crypto module.

Signed-off-by: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Reviewed-by: Janne Peltonen janne.peltonen@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index da5540fe..14b73f95 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -91,7 +91,6 @@ noinst_HEADERS = \ include/odp_classification_inlines.h \ include/odp_classification_internal.h \ include/odp_config_internal.h \ - include/odp_crypto_internal.h \ include/odp_debug_internal.h \ include/odp_errno_define.h \ include/odp_forward_typedefs_internal.h \ diff --git a/platform/linux-generic/include/odp_crypto_internal.h b/platform/linux-generic/include/odp_crypto_internal.h deleted file mode 100644 index 32178d9d..00000000 --- a/platform/linux-generic/include/odp_crypto_internal.h +++ /dev/null @@ -1,73 +0,0 @@ -/* Copyright (c) 2014, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_CRYPTO_INTERNAL_H_ -#define ODP_CRYPTO_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <openssl/evp.h> - -#define MAX_IV_LEN 64 -#define OP_RESULT_MAGIC 0x91919191 - -/** Forward declaration of session structure */ -typedef struct odp_crypto_generic_session odp_crypto_generic_session_t; - -/** - * Algorithm handler function prototype - */ -typedef -odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session); - -/** - * Per crypto session data structure - */ -struct odp_crypto_generic_session { - struct odp_crypto_generic_session *next; - - /* Session creation parameters */ - odp_crypto_session_param_t p; - - odp_bool_t do_cipher_first; - - struct { - /* Copy of session IV data */ - uint8_t iv_data[MAX_IV_LEN]; - uint8_t key_data[EVP_MAX_KEY_LENGTH]; - - const EVP_CIPHER *evp_cipher; - crypto_func_t func; - } cipher; - - struct { - uint8_t key[EVP_MAX_KEY_LENGTH]; - uint32_t key_length; - union { - const EVP_MD *evp_md; - const EVP_CIPHER *evp_cipher; - }; - crypto_func_t func; - } auth; -}; - -/** - * Per session creation operation result - */ -typedef struct odp_crypto_generic_session_result { - odp_crypto_ses_create_err_t rc; - odp_crypto_session_t session; -} odp_crypto_generic_session_result_t; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 2038aa46..3b110c19 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -15,7 +15,6 @@ #include <odp/api/debug.h> #include <odp/api/align.h> #include <odp/api/shared_memory.h> -#include <odp_crypto_internal.h> #include <odp_debug_internal.h> #include <odp/api/hints.h> #include <odp/api/random.h> @@ -89,6 +88,49 @@ static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { static const odp_crypto_auth_capability_t auth_capa_aes_gmac[] = { {.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } };

+/** Forward declaration of session structure */ +typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; + +/** + * Algorithm handler function prototype + */ +typedef +odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); + +/** + * Per crypto session data structure + */ +struct odp_crypto_generic_session_t { + odp_crypto_generic_session_t *next; + + /* Session creation parameters */ + odp_crypto_session_param_t p; + + odp_bool_t do_cipher_first; + + struct { + /* Copy of session IV data */ + uint8_t iv_data[EVP_MAX_IV_LENGTH]; + uint8_t key_data[EVP_MAX_KEY_LENGTH]; + + const EVP_CIPHER *evp_cipher; + crypto_func_t func; + } cipher; + + struct { + uint8_t key[EVP_MAX_KEY_LENGTH]; + uint32_t key_length; + uint32_t bytes; + union { + const EVP_MD *evp_md; + const EVP_CIPHER *evp_cipher; + }; + crypto_func_t func; + } auth; +}; + typedef struct odp_crypto_global_s odp_crypto_global_t;

struct odp_crypto_global_s { @@ -860,7 +902,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param;

- if (session->p.iv.length > MAX_IV_LEN) { + if (session->p.iv.length > EVP_MAX_IV_LENGTH) { ODP_DBG("Maximum IV length exceeded\n"); *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; goto err;

commit 6c4ecdb73aa869e4733b9b323d9dc4c625a63e58 Author: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Date: Thu Dec 14 04:28:47 2017 +0300

test: performance: add AES tests to crypto performance tests

Signed-off-by: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Reviewed-by: Janne Peltonen janne.peltonen@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 761cc8cf..f6524bc6 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -30,7 +30,7 @@ */ #define POOL_NUM_PKT 64

-static uint8_t test_iv[8] = "01234567"; +static uint8_t test_iv[16] = "0123456789abcdef";

static uint8_t test_key16[16] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, @@ -38,6 +38,12 @@ static uint8_t test_key16[16] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x10, };

+static uint8_t test_key20[20] = { 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, +}; + static uint8_t test_key24[24] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, @@ -226,6 +232,69 @@ static crypto_alg_config_t algs_config[] = { .auth_digest_len = 12, }, }, + { + .name = "aes-cbc-null", + .session = { + .cipher_alg = ODP_CIPHER_ALG_AES_CBC, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .iv = { + .data = test_iv, + .length = 16, + }, + .auth_alg = ODP_AUTH_ALG_NULL + }, + }, + { + .name = "aes-cbc-hmac-sha1-96", + .session = { + .cipher_alg = ODP_CIPHER_ALG_AES_CBC, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .iv = { + .data = test_iv, + .length = 16, + }, + .auth_alg = ODP_AUTH_ALG_SHA1_HMAC, + .auth_key = { + .data = test_key20, + .length = sizeof(test_key20) + }, + .auth_digest_len = 12, + }, + }, + { + .name = "null-hmac-sha1-96", + .session = { + .cipher_alg = ODP_CIPHER_ALG_NULL, + .auth_alg = ODP_AUTH_ALG_SHA1_HMAC, + .auth_key = { + .data = test_key20, + .length = sizeof(test_key20) + }, + .auth_digest_len = 12, + }, + }, + { + .name = "aes-gcm", + .session = { + .cipher_alg = ODP_CIPHER_ALG_AES_GCM, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .iv = { + .data = test_iv, + .length = 12, + }, + .auth_alg = ODP_AUTH_ALG_AES_GCM, + .auth_digest_len = 16, + }, + }, };

/**

-----------------------------------------------------------------------

Summary of changes: platform/linux-generic/Makefile.am | 1 - .../linux-generic/include/odp_crypto_internal.h | 73 ----- platform/linux-generic/include/odp_internal.h | 2 + platform/linux-generic/odp_crypto.c | 365 +++++++++++++++------ platform/linux-generic/odp_init.c | 13 + test/performance/odp_crypto.c | 71 +++- 6 files changed, 346 insertions(+), 179 deletions(-) delete mode 100644 platform/linux-generic/include/odp_crypto_internal.h

hooks/post-receive