Skip to content

Commit

Permalink
crypto: inside-secure - remove request list to improve performance
Browse files Browse the repository at this point in the history 
This patch main goal is to improve driver performance by moving the
crypto request from a list to a RDR ring shadow.

This is possible since there is one producer and one consume for this
RDR request shadow and one ring descriptor is left unused.
Doing this change eliminates the use of spinlock when accessing the
descriptor ring and the need to dynamicaly allocate memory per crypto
request.

The crypto request is placed in the first RDR shadow descriptor only
if there are enough descriptors, when the result handler is invoked,
it fetches the first result descriptor from RDR shadow.

Signed-off-by: Ofer Heifetz <[email protected]>
Signed-off-by: Antoine Tenart <[email protected]>
Signed-off-by: Herbert Xu <[email protected]>
  • Loading branch information
@oferhz @herbertx
oferhz authored and herbertx committed Jul 8, 2018
1 parent 6246987 commit 9744fec
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 121 deletions.
91 changes: 45 additions & 46 deletions drivers/crypto/inside-secure/safexcel.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,6 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
{
struct crypto_async_request *req, *backlog;
struct safexcel_context *ctx;
struct safexcel_request *request;
int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;

/* If a request wasn't properly dequeued because of a lack of resources,
Expand All @@ -533,16 +532,10 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
}

handle_req:
request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
if (!request)
goto request_failed;

ctx = crypto_tfm_ctx(req->tfm);
ret = ctx->send(req, ring, request, &commands, &results);
if (ret) {
kfree(request);
ret = ctx->send(req, ring, &commands, &results);
if (ret)
goto request_failed;
}

if (backlog)
backlog->complete(backlog, -EINPROGRESS);
Expand All @@ -551,14 +544,8 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
* to the engine because the input data was cached, continue to
* dequeue other requests as this is valid and not an error.
*/
if (!commands && !results) {
kfree(request);
if (!commands && !results)
continue;
}

spin_lock_bh(&priv->ring[ring].egress_lock);
list_add_tail(&request->list, &priv->ring[ring].list);
spin_unlock_bh(&priv->ring[ring].egress_lock);

cdesc += commands;
rdesc += results;
Expand All @@ -576,7 +563,7 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
if (!nreq)
return;

spin_lock_bh(&priv->ring[ring].egress_lock);
spin_lock_bh(&priv->ring[ring].lock);

priv->ring[ring].requests += nreq;

Expand All @@ -585,7 +572,7 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
priv->ring[ring].busy = true;
}

spin_unlock_bh(&priv->ring[ring].egress_lock);
spin_unlock_bh(&priv->ring[ring].lock);

/* let the RDR know we have pending descriptors */
writel((rdesc * priv->config.rd_offset) << 2,
Expand Down Expand Up @@ -617,6 +604,24 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
return -EINVAL;
}

inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc,
struct crypto_async_request *req)
{
int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);

priv->ring[ring].rdr_req[i] = req;
}

inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
{
int i = safexcel_ring_first_rdr_index(priv, ring);

return priv->ring[ring].rdr_req[i];
}

void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
{
struct safexcel_command_desc *cdesc;
Expand Down Expand Up @@ -645,21 +650,16 @@ void safexcel_inv_complete(struct crypto_async_request *req, int error)

int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring,
struct safexcel_request *request)
dma_addr_t ctxr_dma, int ring)
{
struct safexcel_command_desc *cdesc;
struct safexcel_result_desc *rdesc;
int ret = 0;

spin_lock_bh(&priv->ring[ring].egress_lock);

/* Prepare command descriptor */
cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
if (IS_ERR(cdesc)) {
ret = PTR_ERR(cdesc);
goto unlock;
}
if (IS_ERR(cdesc))
return PTR_ERR(cdesc);

cdesc->control_data.type = EIP197_TYPE_EXTENDED;
cdesc->control_data.options = 0;
Expand All @@ -674,21 +674,20 @@ int safexcel_invalidate_cache(struct crypto_async_request *async,
goto cdesc_rollback;
}

request->req = async;
goto unlock;
safexcel_rdr_req_set(priv, ring, rdesc, async);

return ret;

cdesc_rollback:
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);

unlock:
spin_unlock_bh(&priv->ring[ring].egress_lock);
return ret;
}

static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
int ring)
{
struct safexcel_request *sreq;
struct crypto_async_request *req;
struct safexcel_context *ctx;
int ret, i, nreq, ndesc, tot_descs, handled = 0;
bool should_complete;
Expand All @@ -703,28 +702,22 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
goto requests_left;

for (i = 0; i < nreq; i++) {
spin_lock_bh(&priv->ring[ring].egress_lock);
sreq = list_first_entry(&priv->ring[ring].list,
struct safexcel_request, list);
list_del(&sreq->list);
spin_unlock_bh(&priv->ring[ring].egress_lock);

ctx = crypto_tfm_ctx(sreq->req->tfm);
ndesc = ctx->handle_result(priv, ring, sreq->req,
req = safexcel_rdr_req_get(priv, ring);

ctx = crypto_tfm_ctx(req->tfm);
ndesc = ctx->handle_result(priv, ring, req,
&should_complete, &ret);
if (ndesc < 0) {
kfree(sreq);
dev_err(priv->dev, "failed to handle result (%d)", ndesc);
goto acknowledge;
}

if (should_complete) {
local_bh_disable();
sreq->req->complete(sreq->req, ret);
req->complete(req, ret);
local_bh_enable();
}

kfree(sreq);
tot_descs += ndesc;
handled++;
}
Expand All @@ -743,15 +736,15 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
goto handle_results;

requests_left:
spin_lock_bh(&priv->ring[ring].egress_lock);
spin_lock_bh(&priv->ring[ring].lock);

priv->ring[ring].requests -= handled;
safexcel_try_push_requests(priv, ring);

if (!priv->ring[ring].requests)
priv->ring[ring].busy = false;

spin_unlock_bh(&priv->ring[ring].egress_lock);
spin_unlock_bh(&priv->ring[ring].lock);
}

static void safexcel_dequeue_work(struct work_struct *work)
Expand Down Expand Up @@ -1073,6 +1066,14 @@ static int safexcel_probe(struct platform_device *pdev)
if (ret)
goto err_reg_clk;

priv->ring[i].rdr_req = devm_kzalloc(dev,
sizeof(priv->ring[i].rdr_req) * EIP197_DEFAULT_RING_SIZE,
GFP_KERNEL);
if (!priv->ring[i].rdr_req) {
ret = -ENOMEM;
goto err_reg_clk;
}

ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
if (!ring_irq) {
ret = -ENOMEM;
Expand Down Expand Up @@ -1108,9 +1109,7 @@ static int safexcel_probe(struct platform_device *pdev)
crypto_init_queue(&priv->ring[i].queue,
EIP197_DEFAULT_RING_SIZE);

INIT_LIST_HEAD(&priv->ring[i].list);
spin_lock_init(&priv->ring[i].lock);
spin_lock_init(&priv->ring[i].egress_lock);
spin_lock_init(&priv->ring[i].queue_lock);
}

Expand Down
31 changes: 18 additions & 13 deletions drivers/crypto/inside-secure/safexcel.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,8 +512,7 @@ struct safexcel_desc_ring {
void *write;
void *read;

/* number of elements used in the ring */
unsigned nr;
/* descriptor element offset */
unsigned offset;
};

Expand All @@ -523,11 +522,6 @@ enum safexcel_alg_type {
SAFEXCEL_ALG_TYPE_AHASH,
};

struct safexcel_request {
struct list_head list;
struct crypto_async_request *req;
};

struct safexcel_config {
u32 pes;
u32 rings;
Expand All @@ -547,16 +541,17 @@ struct safexcel_work_data {

struct safexcel_ring {
spinlock_t lock;
spinlock_t egress_lock;

struct list_head list;
struct workqueue_struct *workqueue;
struct safexcel_work_data work_data;

/* command/result rings */
struct safexcel_desc_ring cdr;
struct safexcel_desc_ring rdr;

/* result ring crypto API request */
struct crypto_async_request **rdr_req;

/* queue */
struct crypto_queue queue;
spinlock_t queue_lock;
Expand Down Expand Up @@ -618,8 +613,7 @@ struct safexcel_crypto_priv {

struct safexcel_context {
int (*send)(struct crypto_async_request *req, int ring,
struct safexcel_request *request, int *commands,
int *results);
int *commands, int *results);
int (*handle_result)(struct safexcel_crypto_priv *priv, int ring,
struct crypto_async_request *req, bool *complete,
int *ret);
Expand Down Expand Up @@ -668,14 +662,14 @@ int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring);
int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring,
struct safexcel_request *request);
dma_addr_t ctxr_dma, int ring);
int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *cdr,
struct safexcel_desc_ring *rdr);
int safexcel_select_ring(struct safexcel_crypto_priv *priv);
void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring);
void *safexcel_ring_first_rptr(struct safexcel_crypto_priv *priv, int ring);
void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring);
struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
Expand All @@ -688,6 +682,17 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri
int ring_id,
bool first, bool last,
dma_addr_t data, u32 len);
int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
int ring);
int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc);
void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc,
struct crypto_async_request *req);
inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring);
void safexcel_inv_complete(struct crypto_async_request *req, int error);
int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
void *istate, void *ostate);
Expand Down
Loading

0 comments on commit 9744fec

Please sign in to comment.