mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-16 11:21:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

nvme-tcp: use crc32c() and skb_copy_and_crc32c_datagram_iter()

Browse Source 
Now that the crc32c() library function directly takes advantage of
architecture-specific optimizations and there also now exists a function
skb_copy_and_crc32c_datagram_iter(), it is unnecessary to go through the
crypto_ahash API.  Just use those functions.  This is much simpler, and
it also improves performance due to eliminating the crypto API overhead.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Link: https://patch.msgid.link/20250519175012.36581-10-ebiggers@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Biggers
2025-05-19 10:50:11 -07:00
committed by Jakub Kicinski
parent ea6342d989
commit 427fff9aff
2 changed files with 42 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/nvme/host/Kconfig
View File

@@ -84,9 +84,9 @@ config NVME_TCP
tristate "NVM Express over Fabrics TCP host driver" tristate "NVM Express over Fabrics TCP host driver"
depends on INET depends on INET
depends on BLOCK depends on BLOCK
select CRC32
select NET_CRC32C
select NVME_FABRICS select NVME_FABRICS
select CRYPTO
select CRYPTO_CRC32C
help help
This provides support for the NVMe over Fabrics protocol using This provides support for the NVMe over Fabrics protocol using
the TCP transport. This allows you to use remote block devices the TCP transport. This allows you to use remote block devices

124
drivers/nvme/host/tcp.c
View File

@@ -8,6 +8,7 @@
#include <linux/init.h> #include <linux/init.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/crc32.h>
#include <linux/nvme-tcp.h> #include <linux/nvme-tcp.h>
#include <linux/nvme-keyring.h> #include <linux/nvme-keyring.h>
#include <net/sock.h> #include <net/sock.h>
@@ -16,7 +17,6 @@
#include <net/tls_prot.h> #include <net/tls_prot.h>
#include <net/handshake.h> #include <net/handshake.h>
#include <linux/blk-mq.h> #include <linux/blk-mq.h>
#include <crypto/hash.h>
#include <net/busy_poll.h> #include <net/busy_poll.h>
#include <trace/events/sock.h> #include <trace/events/sock.h>
@@ -168,8 +168,8 @@ struct nvme_tcp_queue {
bool hdr_digest; bool hdr_digest;
bool data_digest; bool data_digest;
bool tls_enabled; bool tls_enabled;
struct ahash_request *rcv_hash; u32 rcv_crc;
struct ahash_request *snd_hash; u32 snd_crc;
__le32 exp_ddgst; __le32 exp_ddgst;
__le32 recv_ddgst; __le32 recv_ddgst;
struct completion tls_complete; struct completion tls_complete;
@@ -456,32 +456,38 @@ nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
return req; return req;
} }
static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, #define NVME_TCP_CRC_SEED (~0)
__le32 *dgst)
static inline void nvme_tcp_ddgst_update(u32 *crcp,
struct page *page, size_t off, size_t len)
{ {
ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); page += off / PAGE_SIZE;
crypto_ahash_final(hash); off %= PAGE_SIZE;
while (len) {
const void *vaddr = kmap_local_page(page);
size_t n = min(len, (size_t)PAGE_SIZE - off);
*crcp = crc32c(*crcp, vaddr + off, n);
kunmap_local(vaddr);
page++;
off = 0;
len -= n;
}
} }
static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, static inline __le32 nvme_tcp_ddgst_final(u32 crc)
struct page *page, off_t off, size_t len)
{ {
struct scatterlist sg; return cpu_to_le32(~crc);
sg_init_table(&sg, 1);
sg_set_page(&sg, page, len, off);
ahash_request_set_crypt(hash, &sg, NULL, len);
crypto_ahash_update(hash);
} }
static inline void nvme_tcp_hdgst(struct ahash_request *hash, static inline __le32 nvme_tcp_hdgst(const void *pdu, size_t len)
void *pdu, size_t len)
{ {
struct scatterlist sg; return cpu_to_le32(~crc32c(NVME_TCP_CRC_SEED, pdu, len));
}
sg_init_one(&sg, pdu, len); static inline void nvme_tcp_set_hdgst(void *pdu, size_t len)
ahash_request_set_crypt(hash, &sg, pdu + len, len); {
crypto_ahash_digest(hash); *(__le32 *)(pdu + len) = nvme_tcp_hdgst(pdu, len);
} }
static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
@@ -499,8 +505,7 @@ static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
} }
recv_digest = *(__le32 *)(pdu + hdr->hlen); recv_digest = *(__le32 *)(pdu + hdr->hlen);
nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); exp_digest = nvme_tcp_hdgst(pdu, pdu_len);
exp_digest = *(__le32 *)(pdu + hdr->hlen);
if (recv_digest != exp_digest) { if (recv_digest != exp_digest) {
dev_err(queue->ctrl->ctrl.device, dev_err(queue->ctrl->ctrl.device,
"header digest error: recv %#x expected %#x\n", "header digest error: recv %#x expected %#x\n",
@@ -526,7 +531,7 @@ static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
nvme_tcp_queue_id(queue)); nvme_tcp_queue_id(queue));
return -EPROTO; return -EPROTO;
} }
crypto_ahash_init(queue->rcv_hash); queue->rcv_crc = NVME_TCP_CRC_SEED;
return 0; return 0;
} }
@@ -926,8 +931,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
iov_iter_count(&req->iter)); iov_iter_count(&req->iter));
if (queue->data_digest) if (queue->data_digest)
ret = skb_copy_and_hash_datagram_iter(skb, *offset, ret = skb_copy_and_crc32c_datagram_iter(skb, *offset,
&req->iter, recv_len, queue->rcv_hash); &req->iter, recv_len, &queue->rcv_crc);
else else
ret = skb_copy_datagram_iter(skb, *offset, ret = skb_copy_datagram_iter(skb, *offset,
&req->iter, recv_len); &req->iter, recv_len);
@@ -945,7 +950,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
if (!queue->data_remaining) { if (!queue->data_remaining) {
if (queue->data_digest) { if (queue->data_digest) {
nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); queue->exp_ddgst = nvme_tcp_ddgst_final(queue->rcv_crc);
queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
} else { } else {
if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
@@ -1147,7 +1152,7 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
return ret; return ret;
if (queue->data_digest) if (queue->data_digest)
nvme_tcp_ddgst_update(queue->snd_hash, page, nvme_tcp_ddgst_update(&queue->snd_crc, page,
offset, ret); offset, ret);
/* /*
@@ -1161,8 +1166,8 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
/* fully successful last send in current PDU */ /* fully successful last send in current PDU */
if (last && ret == len) { if (last && ret == len) {
if (queue->data_digest) { if (queue->data_digest) {
nvme_tcp_ddgst_final(queue->snd_hash, req->ddgst =
&req->ddgst); nvme_tcp_ddgst_final(queue->snd_crc);
req->state = NVME_TCP_SEND_DDGST; req->state = NVME_TCP_SEND_DDGST;
req->offset = 0; req->offset = 0;
} else { } else {
@@ -1194,7 +1199,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
msg.msg_flags |= MSG_EOR; msg.msg_flags |= MSG_EOR;
if (queue->hdr_digest && !req->offset) if (queue->hdr_digest && !req->offset)
nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
bvec_set_virt(&bvec, (void *)pdu + req->offset, len); bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
@@ -1207,7 +1212,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
if (inline_data) { if (inline_data) {
req->state = NVME_TCP_SEND_DATA; req->state = NVME_TCP_SEND_DATA;
if (queue->data_digest) if (queue->data_digest)
crypto_ahash_init(queue->snd_hash); queue->snd_crc = NVME_TCP_CRC_SEED;
} else { } else {
nvme_tcp_done_send_req(queue); nvme_tcp_done_send_req(queue);
} }
@@ -1229,7 +1234,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
int ret; int ret;
if (queue->hdr_digest && !req->offset) if (queue->hdr_digest && !req->offset)
nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
if (!req->h2cdata_left) if (!req->h2cdata_left)
msg.msg_flags |= MSG_SPLICE_PAGES; msg.msg_flags |= MSG_SPLICE_PAGES;
@@ -1244,7 +1249,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
if (!len) { if (!len) {
req->state = NVME_TCP_SEND_DATA; req->state = NVME_TCP_SEND_DATA;
if (queue->data_digest) if (queue->data_digest)
crypto_ahash_init(queue->snd_hash); queue->snd_crc = NVME_TCP_CRC_SEED;
return 1; return 1;
} }
req->offset += ret; req->offset += ret;
@@ -1384,41 +1389,6 @@ static void nvme_tcp_io_work(struct work_struct *w)
queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
} }
static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
ahash_request_free(queue->rcv_hash);
ahash_request_free(queue->snd_hash);
crypto_free_ahash(tfm);
}
static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
{
struct crypto_ahash *tfm;
tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm))
return PTR_ERR(tfm);
queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!queue->snd_hash)
goto free_tfm;
ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!queue->rcv_hash)
goto free_snd_hash;
ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
return 0;
free_snd_hash:
ahash_request_free(queue->snd_hash);
free_tfm:
crypto_free_ahash(tfm);
return -ENOMEM;
}
static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
{ {
struct nvme_tcp_request *async = &ctrl->async_req; struct nvme_tcp_request *async = &ctrl->async_req;
@@ -1451,9 +1421,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return; return;
if (queue->hdr_digest || queue->data_digest)
nvme_tcp_free_crypto(queue);
page_frag_cache_drain(&queue->pf_cache); page_frag_cache_drain(&queue->pf_cache);
noreclaim_flag = memalloc_noreclaim_save(); noreclaim_flag = memalloc_noreclaim_save();
@@ -1867,21 +1834,13 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
queue->hdr_digest = nctrl->opts->hdr_digest; queue->hdr_digest = nctrl->opts->hdr_digest;
queue->data_digest = nctrl->opts->data_digest; queue->data_digest = nctrl->opts->data_digest;
if (queue->hdr_digest || queue->data_digest) {
ret = nvme_tcp_alloc_crypto(queue);
if (ret) {
dev_err(nctrl->device,
"failed to allocate queue %d crypto\n", qid);
goto err_sock;
}
}
rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
nvme_tcp_hdgst_len(queue); nvme_tcp_hdgst_len(queue);
queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
if (!queue->pdu) { if (!queue->pdu) {
ret = -ENOMEM; ret = -ENOMEM;
goto err_crypto; goto err_sock;
} }
dev_dbg(nctrl->device, "connecting queue %d\n", dev_dbg(nctrl->device, "connecting queue %d\n",
@@ -1914,9 +1873,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
kernel_sock_shutdown(queue->sock, SHUT_RDWR); kernel_sock_shutdown(queue->sock, SHUT_RDWR);
err_rcv_pdu: err_rcv_pdu:
kfree(queue->pdu); kfree(queue->pdu);
err_crypto:
if (queue->hdr_digest || queue->data_digest)
nvme_tcp_free_crypto(queue);
err_sock: err_sock:
/* ->sock will be released by fput() */ /* ->sock will be released by fput() */
fput(queue->sock->file); fput(queue->sock->file);