From: Harsh Jain <harsh@chelsio.com>
Date: Tue, 6 Mar 2018 10:37:52 +0530
Subject: crypto: chelsio -Split Hash requests for large scatter gather list
Patch-mainline: v4.17-rc1
Git-commit: 5110e65536f35e854f5d520b913505dbdbe22787
References: bsc#1104270 FATE#325931
Send multiple WRs to H/W when No. of entries received in scatter list
cannot be sent in single request.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/crypto/chelsio/chcr_algo.c | 358 +++++++++++++++++++++++++----------
drivers/crypto/chelsio/chcr_algo.h | 10
drivers/crypto/chelsio/chcr_core.h | 6
drivers/crypto/chelsio/chcr_crypto.h | 32 ++-
4 files changed, 298 insertions(+), 108 deletions(-)
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -131,6 +131,11 @@ static inline int is_ofld_imm(const stru
return (skb->len <= SGE_MAX_WR_LEN);
}
+static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
+{
+ memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
+}
+
static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
unsigned int entlen,
unsigned int skip)
@@ -165,6 +170,7 @@ static inline void chcr_handle_ahash_res
int err)
{
struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
+ struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
int digestsize, updated_digestsize;
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
@@ -172,25 +178,43 @@ static inline void chcr_handle_ahash_res
if (input == NULL)
goto out;
digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
- if (reqctx->is_sg_map)
- chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
- if (reqctx->dma_addr)
- dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr,
- reqctx->dma_len, DMA_TO_DEVICE);
- reqctx->dma_addr = 0;
updated_digestsize = digestsize;
if (digestsize == SHA224_DIGEST_SIZE)
updated_digestsize = SHA256_DIGEST_SIZE;
else if (digestsize == SHA384_DIGEST_SIZE)
updated_digestsize = SHA512_DIGEST_SIZE;
- if (reqctx->result == 1) {
- reqctx->result = 0;
- memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
- digestsize);
- } else {
- memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
- updated_digestsize);
+
+ if (hctx_wr->dma_addr) {
+ dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
+ hctx_wr->dma_len, DMA_TO_DEVICE);
+ hctx_wr->dma_addr = 0;
+ }
+ if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
+ req->nbytes)) {
+ if (hctx_wr->result == 1) {
+ hctx_wr->result = 0;
+ memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
+ digestsize);
+ } else {
+ memcpy(reqctx->partial_hash,
+ input + sizeof(struct cpl_fw6_pld),
+ updated_digestsize);
+
+ }
+ goto unmap;
}
+ memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
+ updated_digestsize);
+
+ err = chcr_ahash_continue(req);
+ if (err)
+ goto unmap;
+ return;
+unmap:
+ if (hctx_wr->is_sg_map)
+ chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
+
+
out:
req->base.complete(&req->base, err);
}
@@ -563,7 +587,6 @@ static void ulptx_walk_add_sg(struct ul
if (!len)
return;
-
while (sg && skip) {
if (sg_dma_len(sg) <= skip) {
skip -= sg_dma_len(sg);
@@ -653,6 +676,35 @@ static int generate_copy_rrkey(struct ab
}
return 0;
}
+
+static int chcr_hash_ent_in_wr(struct scatterlist *src,
+ unsigned int minsg,
+ unsigned int space,
+ unsigned int srcskip)
+{
+ int srclen = 0;
+ int srcsg = minsg;
+ int soffset = 0, sless;
+
+ if (sg_dma_len(src) == srcskip) {
+ src = sg_next(src);
+ srcskip = 0;
+ }
+ while (src && space > (sgl_ent_len[srcsg + 1])) {
+ sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
+ CHCR_SRC_SG_SIZE);
+ srclen += sless;
+ soffset += sless;
+ srcsg++;
+ if (sg_dma_len(src) == (soffset + srcskip)) {
+ src = sg_next(src);
+ soffset = 0;
+ srcskip = 0;
+ }
+ }
+ return srclen;
+}
+
static int chcr_sg_ent_in_wr(struct scatterlist *src,
struct scatterlist *dst,
unsigned int minsg,
@@ -1153,7 +1205,7 @@ static int chcr_handle_cipher_resp(struc
}
if (!reqctx->imm) {
bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
- SPACE_LEFT(ablkctx->enckey_len),
+ CIP_SPACE_LEFT(ablkctx->enckey_len),
reqctx->src_ofst, reqctx->dst_ofst);
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
@@ -1257,7 +1309,7 @@ static int process_cipher(struct ablkcip
if (!reqctx->imm) {
bytes = chcr_sg_ent_in_wr(req->src, req->dst,
MIN_CIPHER_SG,
- SPACE_LEFT(ablkctx->enckey_len),
+ CIP_SPACE_LEFT(ablkctx->enckey_len),
0, 0);
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
@@ -1513,35 +1565,24 @@ static struct sk_buff *create_hash_wr(st
struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
struct chcr_wr *chcr_req;
struct ulptx_sgl *ulptx;
- unsigned int nents = 0, transhdr_len, iopad_alignment = 0;
- unsigned int digestsize = crypto_ahash_digestsize(tfm);
- unsigned int kctx_len = 0, temp = 0;
- u8 hash_size_in_response = 0;
+ unsigned int nents = 0, transhdr_len;
+ unsigned int temp = 0;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
GFP_ATOMIC;
struct adapter *adap = padap(h_ctx(tfm)->dev);
int error = 0;
- iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
- kctx_len = param->alg_prm.result_size + iopad_alignment;
- if (param->opad_needed)
- kctx_len += param->alg_prm.result_size + iopad_alignment;
-
- if (req_ctx->result)
- hash_size_in_response = digestsize;
- else
- hash_size_in_response = param->alg_prm.result_size;
- transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
- req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <=
- SGE_MAX_WR_LEN;
- nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0);
+ transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
+ req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
+ param->sg_len) <= SGE_MAX_WR_LEN;
+ nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
+ CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
nents += param->bfr_len ? 1 : 0;
- transhdr_len += req_ctx->imm ? roundup((param->bfr_len +
- param->sg_len), 16) :
- (sgl_len(nents) * 8);
+ transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
+ param->sg_len, 16) : (sgl_len(nents) * 8);
transhdr_len = roundup(transhdr_len, 16);
- skb = alloc_skb(SGE_MAX_WR_LEN, flags);
+ skb = alloc_skb(transhdr_len, flags);
if (!skb)
return ERR_PTR(-ENOMEM);
chcr_req = __skb_put_zero(skb, transhdr_len);
@@ -1573,33 +1614,33 @@ static struct sk_buff *create_hash_wr(st
chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
param->alg_prm.mk_size, 0,
param->opad_needed,
- ((kctx_len +
+ ((param->kctx_len +
sizeof(chcr_req->key_ctx)) >> 4));
chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
- ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len +
+ ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
DUMMY_BYTES);
if (param->bfr_len != 0) {
- req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev,
- req_ctx->reqbfr, param->bfr_len,
- DMA_TO_DEVICE);
+ req_ctx->hctx_wr.dma_addr =
+ dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
+ param->bfr_len, DMA_TO_DEVICE);
if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
- req_ctx->dma_addr)) {
+ req_ctx->hctx_wr. dma_addr)) {
error = -ENOMEM;
goto err;
}
- req_ctx->dma_len = param->bfr_len;
+ req_ctx->hctx_wr.dma_len = param->bfr_len;
} else {
- req_ctx->dma_addr = 0;
+ req_ctx->hctx_wr.dma_addr = 0;
}
chcr_add_hash_src_ent(req, ulptx, param);
/* Request upto max wr size */
- temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len
- + param->bfr_len) : 0);
+ temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
+ (param->sg_len + param->bfr_len) : 0);
atomic_inc(&adap->chcr_stats.digest_rqst);
- create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm,
- hash_size_in_response, transhdr_len,
+ create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
+ param->hash_size, transhdr_len,
temp, 0);
- req_ctx->skb = skb;
+ req_ctx->hctx_wr.skb = skb;
return skb;
err:
kfree_skb(skb);
@@ -1618,7 +1659,6 @@ static int chcr_ahash_update(struct ahas
int error;
bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
-
u_ctx = ULD_CTX(h_ctx(rtfm));
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
h_ctx(rtfm)->tx_qidx))) {
@@ -1635,17 +1675,26 @@ static int chcr_ahash_update(struct ahas
req_ctx->reqlen += nbytes;
return 0;
}
+ chcr_init_hctx_per_wr(req_ctx);
error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
if (error)
return -ENOMEM;
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ params.kctx_len = roundup(params.alg_prm.result_size, 16);
+ params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
+ HASH_SPACE_LEFT(params.kctx_len), 0);
+ if (params.sg_len > req->nbytes)
+ params.sg_len = req->nbytes;
+ params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
+ req_ctx->reqlen;
params.opad_needed = 0;
params.more = 1;
params.last = 0;
- params.sg_len = nbytes - req_ctx->reqlen;
params.bfr_len = req_ctx->reqlen;
params.scmd1 = 0;
- get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
- req_ctx->result = 0;
+ req_ctx->hctx_wr.srcsg = req->src;
+
+ params.hash_size = params.alg_prm.result_size;
req_ctx->data_len += params.sg_len + params.bfr_len;
skb = create_hash_wr(req, ¶ms);
if (IS_ERR(skb)) {
@@ -1653,6 +1702,7 @@ static int chcr_ahash_update(struct ahas
goto unmap;
}
+ req_ctx->hctx_wr.processed += params.sg_len;
if (remainder) {
/* Swap buffers */
swap(req_ctx->reqbfr, req_ctx->skbfr);
@@ -1690,16 +1740,27 @@ static int chcr_ahash_final(struct ahash
struct uld_ctx *u_ctx = NULL;
u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+ chcr_init_hctx_per_wr(req_ctx);
u_ctx = ULD_CTX(h_ctx(rtfm));
if (is_hmac(crypto_ahash_tfm(rtfm)))
params.opad_needed = 1;
else
params.opad_needed = 0;
params.sg_len = 0;
+ req_ctx->hctx_wr.isfinal = 1;
get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
- req_ctx->result = 1;
+ params.kctx_len = roundup(params.alg_prm.result_size, 16);
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.opad_needed = 1;
+ params.kctx_len *= 2;
+ } else {
+ params.opad_needed = 0;
+ }
+
+ req_ctx->hctx_wr.result = 1;
params.bfr_len = req_ctx->reqlen;
req_ctx->data_len += params.bfr_len + params.sg_len;
+ req_ctx->hctx_wr.srcsg = req->src;
if (req_ctx->reqlen == 0) {
create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
params.last = 0;
@@ -1712,16 +1773,81 @@ static int chcr_ahash_final(struct ahash
params.last = 1;
params.more = 0;
}
+ params.hash_size = crypto_ahash_digestsize(rtfm);
skb = create_hash_wr(req, ¶ms);
if (IS_ERR(skb))
return PTR_ERR(skb);
-
+ req_ctx->reqlen = 0;
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
+static int chcr_ahash_continue(struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
+ struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+ struct uld_ctx *u_ctx = NULL;
+ struct sk_buff *skb;
+ struct hash_wr_param params;
+ u8 bs;
+ int error;
+
+ bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+ u_ctx = ULD_CTX(h_ctx(rtfm));
+ if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+ h_ctx(rtfm)->tx_qidx))) {
+ if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+ return -EBUSY;
+ }
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ params.kctx_len = roundup(params.alg_prm.result_size, 16);
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len *= 2;
+ params.opad_needed = 1;
+ } else {
+ params.opad_needed = 0;
+ }
+ params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
+ HASH_SPACE_LEFT(params.kctx_len),
+ hctx_wr->src_ofst);
+ if ((params.sg_len + hctx_wr->processed) > req->nbytes)
+ params.sg_len = req->nbytes - hctx_wr->processed;
+ if (!hctx_wr->result ||
+ ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len /= 2;
+ params.opad_needed = 0;
+ }
+ params.last = 0;
+ params.more = 1;
+ params.sg_len = rounddown(params.sg_len, bs);
+ params.hash_size = params.alg_prm.result_size;
+ params.scmd1 = 0;
+ } else {
+ params.last = 1;
+ params.more = 0;
+ params.hash_size = crypto_ahash_digestsize(rtfm);
+ params.scmd1 = reqctx->data_len + params.sg_len;
+ }
+ params.bfr_len = 0;
+ reqctx->data_len += params.sg_len;
+ skb = create_hash_wr(req, ¶ms);
+ if (IS_ERR(skb)) {
+ error = PTR_ERR(skb);
+ goto err;
+ }
+ hctx_wr->processed += params.sg_len;
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
+ chcr_send_wr(skb);
+ return 0;
+err:
+ return error;
+}
+
static int chcr_ahash_finup(struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
@@ -1740,37 +1866,59 @@ static int chcr_ahash_finup(struct ahash
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
+ chcr_init_hctx_per_wr(req_ctx);
+ error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
+ if (error)
+ return -ENOMEM;
- if (is_hmac(crypto_ahash_tfm(rtfm)))
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ params.kctx_len = roundup(params.alg_prm.result_size, 16);
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len *= 2;
params.opad_needed = 1;
- else
+ } else {
params.opad_needed = 0;
+ }
- params.sg_len = req->nbytes;
+ params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
+ HASH_SPACE_LEFT(params.kctx_len), 0);
+ if (params.sg_len < req->nbytes) {
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len /= 2;
+ params.opad_needed = 0;
+ }
+ params.last = 0;
+ params.more = 1;
+ params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
+ - req_ctx->reqlen;
+ params.hash_size = params.alg_prm.result_size;
+ params.scmd1 = 0;
+ } else {
+ params.last = 1;
+ params.more = 0;
+ params.sg_len = req->nbytes;
+ params.hash_size = crypto_ahash_digestsize(rtfm);
+ params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
+ params.sg_len;
+ }
params.bfr_len = req_ctx->reqlen;
- get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
req_ctx->data_len += params.bfr_len + params.sg_len;
- req_ctx->result = 1;
+ req_ctx->hctx_wr.result = 1;
+ req_ctx->hctx_wr.srcsg = req->src;
if ((req_ctx->reqlen + req->nbytes) == 0) {
create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
params.last = 0;
params.more = 1;
params.scmd1 = 0;
params.bfr_len = bs;
- } else {
- params.scmd1 = req_ctx->data_len;
- params.last = 1;
- params.more = 0;
}
- error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
- if (error)
- return -ENOMEM;
-
skb = create_hash_wr(req, ¶ms);
if (IS_ERR(skb)) {
error = PTR_ERR(skb);
goto unmap;
}
+ req_ctx->reqlen = 0;
+ req_ctx->hctx_wr.processed += params.sg_len;
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
@@ -1801,21 +1949,42 @@ static int chcr_ahash_digest(struct ahas
return -EBUSY;
}
- if (is_hmac(crypto_ahash_tfm(rtfm)))
- params.opad_needed = 1;
- else
- params.opad_needed = 0;
+ chcr_init_hctx_per_wr(req_ctx);
error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
if (error)
return -ENOMEM;
- params.last = 0;
- params.more = 0;
- params.sg_len = req->nbytes;
- params.bfr_len = 0;
- params.scmd1 = 0;
get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
- req_ctx->result = 1;
+ params.kctx_len = roundup(params.alg_prm.result_size, 16);
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len *= 2;
+ params.opad_needed = 1;
+ } else {
+ params.opad_needed = 0;
+ }
+ params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
+ HASH_SPACE_LEFT(params.kctx_len), 0);
+ if (params.sg_len < req->nbytes) {
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ params.kctx_len /= 2;
+ params.opad_needed = 0;
+ }
+ params.last = 0;
+ params.more = 1;
+ params.scmd1 = 0;
+ params.sg_len = rounddown(params.sg_len, bs);
+ params.hash_size = params.alg_prm.result_size;
+ } else {
+ params.sg_len = req->nbytes;
+ params.hash_size = crypto_ahash_digestsize(rtfm);
+ params.last = 1;
+ params.more = 0;
+ params.scmd1 = req->nbytes + req_ctx->data_len;
+
+ }
+ params.bfr_len = 0;
+ req_ctx->hctx_wr.result = 1;
+ req_ctx->hctx_wr.srcsg = req->src;
req_ctx->data_len += params.bfr_len + params.sg_len;
if (req->nbytes == 0) {
@@ -1829,6 +1998,7 @@ static int chcr_ahash_digest(struct ahas
error = PTR_ERR(skb);
goto unmap;
}
+ req_ctx->hctx_wr.processed += params.sg_len;
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
@@ -1845,11 +2015,10 @@ static int chcr_ahash_export(struct ahas
state->reqlen = req_ctx->reqlen;
state->data_len = req_ctx->data_len;
- state->is_sg_map = 0;
- state->result = 0;
memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
memcpy(state->partial_hash, req_ctx->partial_hash,
CHCR_HASH_MAX_DIGEST_SIZE);
+ chcr_init_hctx_per_wr(state);
return 0;
}
@@ -1862,11 +2031,10 @@ static int chcr_ahash_import(struct ahas
req_ctx->data_len = state->data_len;
req_ctx->reqbfr = req_ctx->bfr1;
req_ctx->skbfr = req_ctx->bfr2;
- req_ctx->is_sg_map = 0;
- req_ctx->result = 0;
memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
memcpy(req_ctx->partial_hash, state->partial_hash,
CHCR_HASH_MAX_DIGEST_SIZE);
+ chcr_init_hctx_per_wr(req_ctx);
return 0;
}
@@ -1963,10 +2131,8 @@ static int chcr_sha_init(struct ahash_re
req_ctx->reqlen = 0;
req_ctx->reqbfr = req_ctx->bfr1;
req_ctx->skbfr = req_ctx->bfr2;
- req_ctx->skb = NULL;
- req_ctx->result = 0;
- req_ctx->is_sg_map = 0;
copy_hash_init_values(req_ctx->partial_hash, digestsize);
+
return 0;
}
@@ -2407,22 +2573,26 @@ void chcr_add_hash_src_ent(struct ahash_
struct ulptx_walk ulp_walk;
struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
- if (reqctx->imm) {
+ if (reqctx->hctx_wr.imm) {
u8 *buf = (u8 *)ulptx;
if (param->bfr_len) {
memcpy(buf, reqctx->reqbfr, param->bfr_len);
buf += param->bfr_len;
}
- sg_pcopy_to_buffer(req->src, sg_nents(req->src),
- buf, param->sg_len, 0);
+
+ sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
+ sg_nents(reqctx->hctx_wr.srcsg), buf,
+ param->sg_len, 0);
} else {
ulptx_walk_init(&ulp_walk, ulptx);
if (param->bfr_len)
ulptx_walk_add_page(&ulp_walk, param->bfr_len,
- &reqctx->dma_addr);
- ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len,
- 0);
+ &reqctx->hctx_wr.dma_addr);
+ ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
+ param->sg_len, reqctx->hctx_wr.src_ofst);
+ reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
+ reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
ulptx_walk_end(&ulp_walk);
}
}
@@ -2439,7 +2609,7 @@ int chcr_hash_dma_map(struct device *dev
DMA_TO_DEVICE);
if (!error)
return -ENOMEM;
- req_ctx->is_sg_map = 1;
+ req_ctx->hctx_wr.is_sg_map = 1;
return 0;
}
@@ -2453,7 +2623,7 @@ void chcr_hash_dma_unmap(struct device *
dma_unmap_sg(dev, req->src, sg_nents(req->src),
DMA_TO_DEVICE);
- req_ctx->is_sg_map = 0;
+ req_ctx->hctx_wr.is_sg_map = 0;
}
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -264,8 +264,10 @@
#define MIN_GCM_SG 1 /* IV */
#define MIN_DIGEST_SG 1 /*Partial Buffer*/
#define MIN_CCM_SG 2 /*IV+B0*/
-#define SPACE_LEFT(len) \
- ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len)))
+#define CIP_SPACE_LEFT(len) \
+ ((SGE_MAX_WR_LEN - CIP_WR_MIN_LEN - (len)))
+#define HASH_SPACE_LEFT(len) \
+ ((SGE_MAX_WR_LEN - HASH_WR_MIN_LEN - (len)))
struct algo_param {
unsigned int auth_mode;
@@ -274,12 +276,14 @@ struct algo_param {
};
struct hash_wr_param {
+ struct algo_param alg_prm;
unsigned int opad_needed;
unsigned int more;
unsigned int last;
- struct algo_param alg_prm;
+ unsigned int kctx_len;
unsigned int sg_len;
unsigned int bfr_len;
+ unsigned int hash_size;
u64 scmd1;
};
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -54,10 +54,14 @@
#define MAC_ERROR_BIT 0
#define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1)
#define MAX_SALT 4
-#define WR_MIN_LEN (sizeof(struct chcr_wr) + \
+#define CIP_WR_MIN_LEN (sizeof(struct chcr_wr) + \
sizeof(struct cpl_rx_phys_dsgl) + \
sizeof(struct ulptx_sgl))
+#define HASH_WR_MIN_LEN (sizeof(struct chcr_wr) + \
+ DUMMY_BYTES + \
+ sizeof(struct ulptx_sgl))
+
#define padap(dev) pci_get_drvdata(dev->u_ctx->lldi.pdev)
struct uld_ctx;
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -258,21 +258,32 @@ struct chcr_context {
struct __crypto_ctx crypto_ctx[0];
};
-struct chcr_ahash_req_ctx {
+struct chcr_hctx_per_wr {
+ struct scatterlist *srcsg;
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ u32 dma_len;
+ unsigned int src_ofst;
+ unsigned int processed;
u32 result;
- u8 bfr1[CHCR_HASH_MAX_BLOCK_SIZE_128];
- u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128];
+ u8 is_sg_map;
+ u8 imm;
+ /*Final callback called. Driver cannot rely on nbytes to decide
+ * final call
+ */
+ u8 isfinal;
+};
+
+struct chcr_ahash_req_ctx {
+ struct chcr_hctx_per_wr hctx_wr;
u8 *reqbfr;
u8 *skbfr;
- dma_addr_t dma_addr;
- u32 dma_len;
+ /* SKB which is being sent to the hardware for processing */
+ u64 data_len; /* Data len till time */
u8 reqlen;
- u8 imm;
- u8 is_sg_map;
u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
- u64 data_len; /* Data len till time */
- /* SKB which is being sent to the hardware for processing */
- struct sk_buff *skb;
+ u8 bfr1[CHCR_HASH_MAX_BLOCK_SIZE_128];
+ u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128];
};
struct chcr_blkcipher_req_ctx {
@@ -329,4 +340,5 @@ void chcr_add_hash_src_ent(struct ahash_
struct hash_wr_param *param);
int chcr_hash_dma_map(struct device *dev, struct ahash_request *req);
void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req);
+static int chcr_ahash_continue(struct ahash_request *req);
#endif /* __CHCR_CRYPTO_H__ */