mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-10 05:39:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'add-af_xdp-support-for-cn10k'

Browse Source 
Suman Ghosh says:

====================
Add af_xdp support for cn10k

This patchset includes changes to support AF_XDP for cn10k chipsets. Both
non-zero copy and zero copy will be supported after these changes. Also,
the RSS will be reconfigured once a particular receive queue is
added/removed to/from AF_XDP support.

Patch #1: octeontx2-pf: use xdp_return_frame() to free xdp buffers

Patch #2: octeontx2-pf: Add AF_XDP non-zero copy support

Patch #3: octeontx2-pf: AF_XDP zero copy receive support

Patch #4: octeontx2-pf: Reconfigure RSS table after enabling AF_XDP
zerocopy on rx queue

Patch #5: octeontx2-pf: Prepare for AF_XDP transmit

Patch #6: octeontx2-pf: AF_XDP zero copy transmit support
====================

Link: https://patch.msgid.link/20250213053141.2833254-1-sumang@marvell.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni
2025-02-18 11:36:29 +01:00
parent 0f375d90c4 53616af09b
commit b4cb730862
12 changed files with 554 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/net/ethernet/marvell/octeontx2/nic/Makefile
View File

@@ -9,7 +9,7 @@ obj-$(CONFIG_RVU_ESWITCH) += rvu_rep.o
rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \
otx2_flows.o otx2_tc.o cn10k.o otx2_dmac_flt.o \
otx2_devlink.o qos_sq.o qos.o
otx2_devlink.o qos_sq.o qos.o otx2_xsk.o
rvu_nicvf-y := otx2_vf.o
rvu_rep-y := rep.o

7
drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
View File

@@ -112,9 +112,12 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
struct otx2_nic *pfvf = dev;
int cnt = cq->pool_ptrs;
u64 ptrs[NPA_MAX_BURST];
struct otx2_pool *pool;
dma_addr_t bufptr;
int num_ptrs = 1;
pool = &pfvf->qset.pool[cq->cq_idx];
/* Refill pool with new buffers */
while (cq->pool_ptrs) {
if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
@@ -124,7 +127,9 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
break;
}
cq->pool_ptrs--;
ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;
ptrs[num_ptrs] = pool->xsk_pool ?
(u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;
num_ptrs++;
if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,

122
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
View File

@@ -17,6 +17,7 @@
#include "otx2_common.h"
#include "otx2_struct.h"
#include "cn10k.h"
#include "otx2_xsk.h"
static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf)
{
@@ -330,6 +331,10 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id)
rss_ctx = rss->rss_ctx[ctx_id];
/* Get memory to put this msg */
for (idx = 0; idx < rss->rss_size; idx++) {
/* Ignore the queue if AF_XDP zero copy is enabled */
if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
continue;
aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
if (!aq) {
/* The shared memory buffer can be full.
@@ -549,10 +554,13 @@ static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}
static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma)
dma_addr_t *dma, int qidx, int idx)
{
u8 *buf;
if (pool->xsk_pool)
return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx);
if (pool->page_pool)
return otx2_alloc_pool_buf(pfvf, pool, dma);
@@ -571,12 +579,12 @@ static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma)
dma_addr_t *dma, int qidx, int idx)
{
int ret;
local_bh_disable();
ret = __otx2_alloc_rbuf(pfvf, pool, dma);
ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx);
local_bh_enable();
return ret;
}
@@ -584,7 +592,8 @@ int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq,
dma_addr_t *dma)
{
if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma)))
if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma,
cq->cq_idx, cq->pool_ptrs - 1)))
return -ENOMEM;
return 0;
}
@@ -884,7 +893,7 @@ void otx2_sqb_flush(struct otx2_nic *pfvf)
#define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */
#define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */
static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
{
struct otx2_qset *qset = &pfvf->qset;
struct nix_aq_enq_req *aq;
@@ -1028,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
sq->stats.bytes = 0;
sq->stats.pkts = 0;
/* Attach XSK_BUFF_POOL to XDP queue */
if (qidx > pfvf->hw.xdp_queues)
otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));
chan_offset = qidx % pfvf->hw.tx_chan_cnt;
err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
@@ -1041,12 +1054,13 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
}
static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
{
struct otx2_qset *qset = &pfvf->qset;
int err, pool_id, non_xdp_queues;
struct nix_aq_enq_req *aq;
struct otx2_cq_queue *cq;
struct otx2_pool *pool;
cq = &qset->cq[qidx];
cq->cq_idx = qidx;
@@ -1055,8 +1069,20 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
cq->cq_type = CQ_RX;
cq->cint_idx = qidx;
cq->cqe_cnt = qset->rqe_cnt;
if (pfvf->xdp_prog)
if (pfvf->xdp_prog) {
xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0);
pool = &qset->pool[qidx];
if (pool->xsk_pool) {
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
MEM_TYPE_XSK_BUFF_POOL,
NULL);
xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq);
} else if (pool->page_pool) {
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
MEM_TYPE_PAGE_POOL,
pool->page_pool);
}
}
} else if (qidx < non_xdp_queues) {
cq->cq_type = CQ_TX;
cq->cint_idx = qidx - pfvf->hw.rx_queues;
@@ -1275,9 +1301,10 @@ void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool,
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_head_page(phys_to_virt(pa));
if (pool->page_pool) {
page_pool_put_full_page(pool->page_pool, page, true);
} else if (pool->xsk_pool) {
/* Note: No way of identifying xdp_buff */
} else {
dma_unmap_page_attrs(pfvf->dev, iova, size,
DMA_FROM_DEVICE,
@@ -1292,6 +1319,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
int pool_id, pool_start = 0, pool_end = 0, size = 0;
struct otx2_pool *pool;
u64 iova;
int idx;
if (type == AURA_NIX_SQ) {
pool_start = otx2_get_pool_idx(pfvf, type, 0);
@@ -1306,16 +1334,21 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
/* Free SQB and RQB pointers from the aura pool */
for (pool_id = pool_start; pool_id < pool_end; pool_id++) {
iova = otx2_aura_allocptr(pfvf, pool_id);
pool = &pfvf->qset.pool[pool_id];
iova = otx2_aura_allocptr(pfvf, pool_id);
while (iova) {
if (type == AURA_NIX_RQ)
iova -= OTX2_HEAD_ROOM;
otx2_free_bufs(pfvf, pool, iova, size);
iova = otx2_aura_allocptr(pfvf, pool_id);
}
for (idx = 0 ; idx < pool->xdp_cnt; idx++) {
if (!pool->xdp[idx])
continue;
xsk_buff_free(pool->xdp[idx]);
}
}
}
@@ -1332,7 +1365,8 @@ void otx2_aura_pool_free(struct otx2_nic *pfvf)
qmem_free(pfvf->dev, pool->stack);
qmem_free(pfvf->dev, pool->fc_addr);
page_pool_destroy(pool->page_pool);
pool->page_pool = NULL;
devm_kfree(pfvf->dev, pool->xdp);
pool->xsk_pool = NULL;
}
devm_kfree(pfvf->dev, pfvf->qset.pool);
pfvf->qset.pool = NULL;
@@ -1419,6 +1453,7 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
int stack_pages, int numptrs, int buf_size, int type)
{
struct page_pool_params pp_params = { 0 };
struct xsk_buff_pool *xsk_pool;
struct npa_aq_enq_req *aq;
struct otx2_pool *pool;
int err;
@@ -1462,21 +1497,35 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
aq->ctype = NPA_AQ_CTYPE_POOL;
aq->op = NPA_AQ_INSTOP_INIT;
if (type != AURA_NIX_RQ) {
pool->page_pool = NULL;
if (type != AURA_NIX_RQ)
return 0;
if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) {
pp_params.order = get_order(buf_size);
pp_params.flags = PP_FLAG_DMA_MAP;
pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
pp_params.nid = NUMA_NO_NODE;
pp_params.dev = pfvf->dev;
pp_params.dma_dir = DMA_FROM_DEVICE;
pool->page_pool = page_pool_create(&pp_params);
if (IS_ERR(pool->page_pool)) {
netdev_err(pfvf->netdev, "Creation of page pool failed\n");
return PTR_ERR(pool->page_pool);
}
return 0;
}
pp_params.order = get_order(buf_size);
pp_params.flags = PP_FLAG_DMA_MAP;
pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
pp_params.nid = NUMA_NO_NODE;
pp_params.dev = pfvf->dev;
pp_params.dma_dir = DMA_FROM_DEVICE;
pool->page_pool = page_pool_create(&pp_params);
if (IS_ERR(pool->page_pool)) {
netdev_err(pfvf->netdev, "Creation of page pool failed\n");
return PTR_ERR(pool->page_pool);
/* Set XSK pool to support AF_XDP zero-copy */
xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id);
if (xsk_pool) {
pool->xsk_pool = xsk_pool;
pool->xdp_cnt = numptrs;
pool->xdp = devm_kcalloc(pfvf->dev,
numptrs, sizeof(struct xdp_buff *), GFP_KERNEL);
if (IS_ERR(pool->xdp)) {
netdev_err(pfvf->netdev, "Creation of xsk pool failed\n");
return PTR_ERR(pool->xdp);
}
}
return 0;
@@ -1537,9 +1586,18 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf)
}
for (ptr = 0; ptr < num_sqbs; ptr++) {
err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
if (err)
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err) {
if (pool->xsk_pool) {
ptr--;
while (ptr >= 0) {
xsk_buff_free(pool->xdp[ptr]);
ptr--;
}
}
goto err_mem;
}
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);
sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr;
}
@@ -1589,11 +1647,19 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf)
/* Allocate pointers and free them to aura/pool */
for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) {
pool = &pfvf->qset.pool[pool_id];
for (ptr = 0; ptr < num_ptrs; ptr++) {
err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
if (err)
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err) {
if (pool->xsk_pool) {
while (ptr)
xsk_buff_free(pool->xdp[--ptr]);
}
return -ENOMEM;
}
pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
pool->xsk_pool ? bufptr :
bufptr + OTX2_HEAD_ROOM);
}
}

17
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
View File

@@ -21,6 +21,7 @@
#include <linux/time64.h>
#include <linux/dim.h>
#include <uapi/linux/if_macsec.h>
#include <net/page_pool/helpers.h>
#include <mbox.h>
#include <npc.h>
@@ -128,6 +129,12 @@ enum otx2_errcodes_re {
ERRCODE_IL4_CSUM = 0x22,
};
enum otx2_xdp_action {
OTX2_XDP_TX = BIT(0),
OTX2_XDP_REDIRECT = BIT(1),
OTX2_AF_XDP_FRAME = BIT(2),
};
struct otx2_dev_stats {
u64 rx_bytes;
u64 rx_frames;
@@ -531,6 +538,8 @@ struct otx2_nic {
/* Inline ipsec */
struct cn10k_ipsec ipsec;
/* af_xdp zero-copy */
unsigned long *af_xdp_zc_qidx;
};
static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
@@ -1002,7 +1011,7 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq);
void otx2_free_pending_sqe(struct otx2_nic *pfvf);
void otx2_sqb_flush(struct otx2_nic *pfvf);
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma);
dma_addr_t *dma, int qidx, int idx);
int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);
void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);
int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
@@ -1032,6 +1041,8 @@ void otx2_pfaf_mbox_destroy(struct otx2_nic *pf);
void otx2_disable_mbox_intr(struct otx2_nic *pf);
void otx2_disable_napi(struct otx2_nic *pf);
irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq);
int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura);
int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx);
/* RSS configuration APIs*/
int otx2_rss_init(struct otx2_nic *pfvf);
@@ -1094,7 +1105,8 @@ int otx2_del_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_add_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable);
int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf);
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
u64 iova, int len, u16 qidx, u16 flags);
u16 otx2_get_max_mtu(struct otx2_nic *pfvf);
int otx2_handle_ntuple_tc_features(struct net_device *netdev,
netdev_features_t features);
@@ -1175,4 +1187,5 @@ static inline int mcam_entry_cmp(const void *a, const void *b)
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
struct sk_buff *skb, int seg, int *len);
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);
int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx);
#endif /* OTX2_COMMON_H */

6
drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
View File

@@ -910,8 +910,12 @@ static int otx2_get_rxfh(struct net_device *dev,
return -ENOENT;
if (indir) {
for (idx = 0; idx < rss->rss_size; idx++)
for (idx = 0; idx < rss->rss_size; idx++) {
/* Ignore if the rx queue is AF_XDP zero copy enabled */
if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
continue;
indir[idx] = rss_ctx->ind_tbl[idx];
}
}
if (rxfh->key)
memcpy(rxfh->key, rss->key, sizeof(rss->key));

32
drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
View File

@@ -27,6 +27,7 @@
#include "qos.h"
#include <rvu_trace.h>
#include "cn10k_ipsec.h"
#include "otx2_xsk.h"
#define DRV_NAME "rvu_nicpf"
#define DRV_STRING "Marvell RVU NIC Physical Function Driver"
@@ -1662,9 +1663,7 @@ void otx2_free_hw_resources(struct otx2_nic *pf)
struct nix_lf_free_req *free_req;
struct mbox *mbox = &pf->mbox;
struct otx2_cq_queue *cq;
struct otx2_pool *pool;
struct msg_req *req;
int pool_id;
int qidx;
/* Ensure all SQE are processed */
@@ -1705,13 +1704,6 @@ void otx2_free_hw_resources(struct otx2_nic *pf)
/* Free RQ buffer pointers*/
otx2_free_aura_ptr(pf, AURA_NIX_RQ);
for (qidx = 0; qidx < pf->hw.rx_queues; qidx++) {
pool_id = otx2_get_pool_idx(pf, AURA_NIX_RQ, qidx);
pool = &pf->qset.pool[pool_id];
page_pool_destroy(pool->page_pool);
pool->page_pool = NULL;
}
otx2_free_cq_res(pf);
/* Free all ingress bandwidth profiles allocated */
@@ -2691,7 +2683,6 @@ static int otx2_get_vf_config(struct net_device *netdev, int vf,
static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
int qidx)
{
struct page *page;
u64 dma_addr;
int err = 0;
@@ -2701,11 +2692,11 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
if (dma_mapping_error(pf->dev, dma_addr))
return -ENOMEM;
err = otx2_xdp_sq_append_pkt(pf, dma_addr, xdpf->len, qidx);
err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
qidx, OTX2_XDP_REDIRECT);
if (!err) {
otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
page = virt_to_page(xdpf->data);
put_page(page);
xdp_return_frame(xdpf);
return -ENOMEM;
}
return 0;
@@ -2789,6 +2780,8 @@ static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
switch (xdp->command) {
case XDP_SETUP_PROG:
return otx2_xdp_setup(pf, xdp->prog);
case XDP_SETUP_XSK_POOL:
return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id);
default:
return -EINVAL;
}
@@ -2866,6 +2859,7 @@ static const struct net_device_ops otx2_netdev_ops = {
.ndo_set_vf_vlan = otx2_set_vf_vlan,
.ndo_get_vf_config = otx2_get_vf_config,
.ndo_bpf = otx2_xdp,
.ndo_xsk_wakeup = otx2_xsk_wakeup,
.ndo_xdp_xmit = otx2_xdp_xmit,
.ndo_setup_tc = otx2_setup_tc,
.ndo_set_vf_trust = otx2_ndo_set_vf_trust,
@@ -3204,16 +3198,26 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
/* Enable link notifications */
otx2_cgx_config_linkevents(pf, true);
pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
if (!pf->af_xdp_zc_qidx) {
err = -ENOMEM;
goto err_sriov_cleannup;
}
#ifdef CONFIG_DCB
err = otx2_dcbnl_set_ops(netdev);
if (err)
goto err_pf_sriov_init;
goto err_free_zc_bmap;
#endif
otx2_qos_init(pf, qos_txqs);
return 0;
err_free_zc_bmap:
bitmap_free(pf->af_xdp_zc_qidx);
err_sriov_cleannup:
otx2_sriov_vfcfg_cleanup(pf);
err_pf_sriov_init:
otx2_shutdown_tc(pf);
err_mcam_flow_del:

188
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
View File

@@ -12,6 +12,7 @@
#include <linux/bpf_trace.h>
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
#include <net/xdp.h>
#include "otx2_reg.h"
#include "otx2_common.h"
@@ -19,6 +20,7 @@
#include "otx2_txrx.h"
#include "otx2_ptp.h"
#include "cn10k.h"
#include "otx2_xsk.h"
#define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
#define PTP_PORT 0x13F
@@ -29,6 +31,12 @@
DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
static int otx2_get_free_sqe(struct otx2_snd_queue *sq)
{
return (sq->cons_head - sq->head - 1 + sq->sqe_cnt)
& (sq->sqe_cnt - 1);
}
static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
struct bpf_prog *prog,
struct nix_cqe_rx_s *cqe,
@@ -96,20 +104,22 @@ static unsigned int frag_num(unsigned int i)
static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
struct otx2_snd_queue *sq,
struct nix_cqe_tx_s *cqe)
struct nix_cqe_tx_s *cqe,
int *xsk_frames)
{
struct nix_send_comp_s *snd_comp = &cqe->comp;
struct sg_list *sg;
struct page *page;
u64 pa;
sg = &sq->sg[snd_comp->sqe_id];
if (sg->flags & OTX2_AF_XDP_FRAME) {
(*xsk_frames)++;
return;
}
pa = otx2_iova_to_phys(pfvf->iommu_domain, sg->dma_addr[0]);
otx2_dma_unmap_page(pfvf, sg->dma_addr[0],
sg->size[0], DMA_TO_DEVICE);
page = virt_to_page(phys_to_virt(pa));
put_page(page);
if (sg->flags & OTX2_XDP_REDIRECT)
otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
xdp_return_frame((struct xdp_frame *)sg->skb);
sg->skb = (u64)NULL;
}
static void otx2_snd_pkt_handler(struct otx2_nic *pfvf,
@@ -431,6 +441,18 @@ int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
return cnt - cq->pool_ptrs;
}
static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
int *xsk_frames, int qidx, int budget)
{
if (*xsk_frames)
xsk_tx_completed(xsk_pool, *xsk_frames);
if (xsk_uses_need_wakeup(xsk_pool))
xsk_set_tx_need_wakeup(xsk_pool);
otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
}
static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
struct otx2_cq_queue *cq, int budget)
{
@@ -439,16 +461,22 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
struct nix_cqe_tx_s *cqe;
struct net_device *ndev;
int processed_cqe = 0;
int xsk_frames = 0;
qidx = cq->cq_idx - pfvf->hw.rx_queues;
sq = &pfvf->qset.sq[qidx];
if (cq->pend_cqe >= budget)
goto process_cqe;
if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
if (sq->xsk_pool)
otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
qidx, budget);
return 0;
}
process_cqe:
qidx = cq->cq_idx - pfvf->hw.rx_queues;
sq = &pfvf->qset.sq[qidx];
while (likely(processed_cqe < budget) && cq->pend_cqe) {
cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
@@ -458,10 +486,8 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
break;
}
qidx = cq->cq_idx - pfvf->hw.rx_queues;
if (cq->cq_type == CQ_XDP)
otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);
otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
else
otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
cqe, budget, &tx_pkts, &tx_bytes);
@@ -502,6 +528,10 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
netif_carrier_ok(ndev))
netif_tx_wake_queue(txq);
}
if (sq->xsk_pool)
otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
return 0;
}
@@ -527,9 +557,10 @@ static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_p
int otx2_napi_handler(struct napi_struct *napi, int budget)
{
struct otx2_cq_queue *rx_cq = NULL;
struct otx2_cq_queue *cq = NULL;
struct otx2_pool *pool = NULL;
struct otx2_cq_poll *cq_poll;
int workdone = 0, cq_idx, i;
struct otx2_cq_queue *cq;
struct otx2_qset *qset;
struct otx2_nic *pfvf;
int filled_cnt = -1;
@@ -554,6 +585,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
if (rx_cq && rx_cq->pool_ptrs)
filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
/* Clear the IRQ */
otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
@@ -566,20 +598,31 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
otx2_adjust_adaptive_coalese(pfvf, cq_poll);
if (likely(cq))
pool = &pfvf->qset.pool[cq->cq_idx];
if (unlikely(!filled_cnt)) {
struct refill_work *work;
struct delayed_work *dwork;
work = &pfvf->refill_wrk[cq->cq_idx];
dwork = &work->pool_refill_work;
/* Schedule a task if no other task is running */
if (!cq->refill_task_sched) {
work->napi = napi;
cq->refill_task_sched = true;
schedule_delayed_work(dwork,
msecs_to_jiffies(100));
if (likely(cq)) {
work = &pfvf->refill_wrk[cq->cq_idx];
dwork = &work->pool_refill_work;
/* Schedule a task if no other task is running */
if (!cq->refill_task_sched) {
work->napi = napi;
cq->refill_task_sched = true;
schedule_delayed_work(dwork,
msecs_to_jiffies(100));
}
/* Call wake-up for not able to fill buffers */
if (pool->xsk_pool)
xsk_set_rx_need_wakeup(pool->xsk_pool);
}
} else {
/* Clear wake-up, since buffers are filled successfully */
if (pool && pool->xsk_pool)
xsk_clear_rx_need_wakeup(pool->xsk_pool);
/* Re-enable interrupts */
otx2_write64(pfvf,
NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
@@ -1147,7 +1190,7 @@ bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq,
/* Check if there is enough room between producer
* and consumer index.
*/
free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1);
free_desc = otx2_get_free_sqe(sq);
if (free_desc < sq->sqe_thresh)
return false;
@@ -1230,15 +1273,19 @@ void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int q
u16 pool_id;
u64 iova;
if (pfvf->xdp_prog)
pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
pool = &pfvf->qset.pool[pool_id];
if (pfvf->xdp_prog) {
if (pool->page_pool)
xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq);
xdp_rxq_info_unreg(&cq->xdp_rxq);
}
if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
return;
pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
pool = &pfvf->qset.pool[pool_id];
while (cq->pend_cqe) {
cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
processed_cqe++;
@@ -1359,8 +1406,9 @@ void otx2_free_pending_sqe(struct otx2_nic *pfvf)
}
}
static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
int len, int *offset)
static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq,
struct xdp_frame *xdpf,
u64 dma_addr, int len, int *offset, u16 flags)
{
struct nix_sqe_sg_s *sg = NULL;
u64 *iova = NULL;
@@ -1377,16 +1425,34 @@ static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
sq->sg[sq->head].dma_addr[0] = dma_addr;
sq->sg[sq->head].size[0] = len;
sq->sg[sq->head].num_segs = 1;
sq->sg[sq->head].flags = flags;
sq->sg[sq->head].skb = (u64)xdpf;
}
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx)
int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx)
{
struct otx2_snd_queue *sq;
int free_sqe;
sq = &pfvf->qset.sq[qidx];
free_sqe = otx2_get_free_sqe(sq);
if (free_sqe < sq->sqe_thresh) {
netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx);
return 0;
}
return free_sqe - sq->sqe_thresh;
}
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
u64 iova, int len, u16 qidx, u16 flags)
{
struct nix_sqe_hdr_s *sqe_hdr;
struct otx2_snd_queue *sq;
int offset, free_sqe;
sq = &pfvf->qset.sq[qidx];
free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb;
free_sqe = otx2_get_free_sqe(sq);
if (free_sqe < sq->sqe_thresh)
return false;
@@ -1405,7 +1471,7 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx)
offset = sizeof(*sqe_hdr);
otx2_xdp_sqe_add_sg(sq, iova, len, &offset);
otx2_xdp_sqe_add_sg(sq, xdpf, iova, len, &offset, flags);
sqe_hdr->sizem1 = (offset / 16) - 1;
pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
@@ -1418,14 +1484,28 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
struct otx2_cq_queue *cq,
bool *need_xdp_flush)
{
struct xdp_buff xdp, *xsk_buff = NULL;
unsigned char *hard_start;
struct otx2_pool *pool;
struct xdp_frame *xdpf;
int qidx = cq->cq_idx;
struct xdp_buff xdp;
struct page *page;
u64 iova, pa;
u32 act;
int err;
pool = &pfvf->qset.pool[qidx];
if (pool->xsk_pool) {
xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
if (!xsk_buff)
return false;
xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
act = bpf_prog_run_xdp(prog, xsk_buff);
goto handle_xdp_verdict;
}
iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_page(phys_to_virt(pa));
@@ -1438,37 +1518,59 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
act = bpf_prog_run_xdp(prog, &xdp);
handle_xdp_verdict:
switch (act) {
case XDP_PASS:
break;
case XDP_TX:
qidx += pfvf->hw.tx_queues;
cq->pool_ptrs++;
return otx2_xdp_sq_append_pkt(pfvf, iova,
cqe->sg.seg_size, qidx);
xdpf = xdp_convert_buff_to_frame(&xdp);
return otx2_xdp_sq_append_pkt(pfvf, xdpf,
cqe->sg.seg_addr,
cqe->sg.seg_size,
qidx, OTX2_XDP_TX);
case XDP_REDIRECT:
cq->pool_ptrs++;
err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
if (xsk_buff) {
err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog);
if (!err) {
*need_xdp_flush = true;
return true;
}
return false;
}
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
if (!err) {
*need_xdp_flush = true;
return true;
}
put_page(page);
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
xdpf = xdp_convert_buff_to_frame(&xdp);
xdp_return_frame(xdpf);
break;
default:
bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
break;
case XDP_ABORTED:
if (xsk_buff)
xsk_buff_free(xsk_buff);
trace_xdp_exception(pfvf->netdev, prog, act);
break;
case XDP_DROP:
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
put_page(page);
cq->pool_ptrs++;
if (xsk_buff) {
xsk_buff_free(xsk_buff);
} else if (page->pp) {
page_pool_recycle_direct(pool->page_pool, page);
} else {
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
put_page(page);
}
return true;
}
return false;

9
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
View File

@@ -12,6 +12,7 @@
#include <linux/iommu.h>
#include <linux/if_vlan.h>
#include <net/xdp.h>
#include <net/xdp_sock_drv.h>
#define LBK_CHAN_BASE 0x000
#define SDP_CHAN_BASE 0x700
@@ -76,6 +77,7 @@ struct otx2_rcv_queue {
struct sg_list {
u16 num_segs;
u16 flags;
u64 skb;
u64 size[OTX2_MAX_FRAGS_IN_SQE];
u64 dma_addr[OTX2_MAX_FRAGS_IN_SQE];
@@ -104,6 +106,8 @@ struct otx2_snd_queue {
/* SQE ring and CPT response queue for Inline IPSEC */
struct qmem *sqe_ring;
struct qmem *cpt_resp;
/* Buffer pool for af_xdp zero-copy */
struct xsk_buff_pool *xsk_pool;
} ____cacheline_aligned_in_smp;
enum cq_type {
@@ -127,7 +131,11 @@ struct otx2_pool {
struct qmem *stack;
struct qmem *fc_addr;
struct page_pool *page_pool;
struct xsk_buff_pool *xsk_pool;
struct xdp_buff **xdp;
u16 xdp_cnt;
u16 rbsize;
u16 xdp_top;
};
struct otx2_cq_queue {
@@ -144,6 +152,7 @@ struct otx2_cq_queue {
void *cqe_base;
struct qmem *cqe;
struct otx2_pool *rbpool;
bool xsk_zc_en;
struct xdp_rxq_info xdp_rxq;
} ____cacheline_aligned_in_smp;

12
drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
View File

@@ -722,15 +722,25 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (err)
goto err_shutdown_tc;
vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
if (!vf->af_xdp_zc_qidx) {
err = -ENOMEM;
goto err_unreg_devlink;
}
#ifdef CONFIG_DCB
err = otx2_dcbnl_set_ops(netdev);
if (err)
goto err_shutdown_tc;
goto err_free_zc_bmap;
#endif
otx2_qos_init(vf, qos_txqs);
return 0;
err_free_zc_bmap:
bitmap_free(vf->af_xdp_zc_qidx);
err_unreg_devlink:
otx2_unregister_dl(vf);
err_shutdown_tc:
otx2_shutdown_tc(vf);
err_unreg_netdev:

225
drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c Normal file
View File

@@ -0,0 +1,225 @@
// SPDX-License-Identifier: GPL-2.0
/* Marvell RVU Ethernet driver
*
* Copyright (C) 2024 Marvell.
*
*/
#include <linux/bpf_trace.h>
#include <linux/stringify.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "otx2_common.h"
#include "otx2_xsk.h"
int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma, int idx)
{
struct xdp_buff *xdp;
int delta;
xdp = xsk_buff_alloc(pool->xsk_pool);
if (!xdp)
return -ENOMEM;
pool->xdp[pool->xdp_top++] = xdp;
*dma = OTX2_DATA_ALIGN(xsk_buff_xdp_get_dma(xdp));
/* Adjust xdp->data for unaligned addresses */
delta = *dma - xsk_buff_xdp_get_dma(xdp);
xdp->data += delta;
return 0;
}
static int otx2_xsk_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id)
{
struct nix_cn10k_aq_enq_req *cn10k_rq_aq;
struct npa_aq_enq_req *aura_aq;
struct npa_aq_enq_req *pool_aq;
struct nix_aq_enq_req *rq_aq;
if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
cn10k_rq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
if (!cn10k_rq_aq)
return -ENOMEM;
cn10k_rq_aq->qidx = qidx;
cn10k_rq_aq->rq.ena = 0;
cn10k_rq_aq->rq_mask.ena = 1;
cn10k_rq_aq->ctype = NIX_AQ_CTYPE_RQ;
cn10k_rq_aq->op = NIX_AQ_INSTOP_WRITE;
} else {
rq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox);
if (!rq_aq)
return -ENOMEM;
rq_aq->qidx = qidx;
rq_aq->sq.ena = 0;
rq_aq->sq_mask.ena = 1;
rq_aq->ctype = NIX_AQ_CTYPE_RQ;
rq_aq->op = NIX_AQ_INSTOP_WRITE;
}
aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
if (!aura_aq)
goto fail;
aura_aq->aura_id = aura_id;
aura_aq->aura.ena = 0;
aura_aq->aura_mask.ena = 1;
aura_aq->ctype = NPA_AQ_CTYPE_AURA;
aura_aq->op = NPA_AQ_INSTOP_WRITE;
pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
if (!pool_aq)
goto fail;
pool_aq->aura_id = aura_id;
pool_aq->pool.ena = 0;
pool_aq->pool_mask.ena = 1;
pool_aq->ctype = NPA_AQ_CTYPE_POOL;
pool_aq->op = NPA_AQ_INSTOP_WRITE;
return otx2_sync_mbox_msg(&pfvf->mbox);
fail:
otx2_mbox_reset(&pfvf->mbox.mbox, 0);
return -ENOMEM;
}
static void otx2_clean_up_rq(struct otx2_nic *pfvf, int qidx)
{
struct otx2_qset *qset = &pfvf->qset;
struct otx2_cq_queue *cq;
struct otx2_pool *pool;
u64 iova;
/* If the DOWN flag is set SQs are already freed */
if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
return;
cq = &qset->cq[qidx];
if (cq)
otx2_cleanup_rx_cqes(pfvf, cq, qidx);
pool = &pfvf->qset.pool[qidx];
iova = otx2_aura_allocptr(pfvf, qidx);
while (iova) {
iova -= OTX2_HEAD_ROOM;
otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
iova = otx2_aura_allocptr(pfvf, qidx);
}
mutex_lock(&pfvf->mbox.lock);
otx2_xsk_ctx_disable(pfvf, qidx, qidx);
mutex_unlock(&pfvf->mbox.lock);
}
int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
{
u16 rx_queues = pf->hw.rx_queues;
u16 tx_queues = pf->hw.tx_queues;
int err;
if (qidx >= rx_queues || qidx >= tx_queues)
return -EINVAL;
err = xsk_pool_dma_map(pool, pf->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
if (err)
return err;
set_bit(qidx, pf->af_xdp_zc_qidx);
otx2_clean_up_rq(pf, qidx);
/* Reconfigure RSS table as 'qidx' cannot be part of RSS now */
otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP);
/* Kick start the NAPI context so that receiving will start */
return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX);
}
int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
{
struct net_device *netdev = pf->netdev;
struct xsk_buff_pool *pool;
struct otx2_snd_queue *sq;
pool = xsk_get_pool_from_qid(netdev, qidx);
if (!pool)
return -EINVAL;
sq = &pf->qset.sq[qidx + pf->hw.tx_queues];
sq->xsk_pool = NULL;
otx2_clean_up_rq(pf, qidx);
clear_bit(qidx, pf->af_xdp_zc_qidx);
xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
/* Reconfigure RSS table as 'qidx' now need to be part of RSS now */
otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP);
return 0;
}
int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
{
if (pool)
return otx2_xsk_pool_enable(pf, pool, qidx);
return otx2_xsk_pool_disable(pf, qidx);
}
int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
{
struct otx2_nic *pf = netdev_priv(dev);
struct otx2_cq_poll *cq_poll = NULL;
struct otx2_qset *qset = &pf->qset;
if (pf->flags & OTX2_FLAG_INTF_DOWN)
return -ENETDOWN;
if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues)
return -EINVAL;
cq_poll = &qset->napi[queue_id];
if (!cq_poll)
return -EINVAL;
/* Trigger interrupt */
if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) {
otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0));
}
return 0;
}
void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx)
{
if (test_bit(qidx, pfvf->af_xdp_zc_qidx))
sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx);
}
void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
int queue, int budget)
{
struct xdp_desc *xdp_desc = pool->tx_descs;
int err, i, work_done = 0, batch;
budget = min(budget, otx2_read_free_sqe(pfvf, queue));
batch = xsk_tx_peek_release_desc_batch(pool, budget);
if (!batch)
return;
for (i = 0; i < batch; i++) {
dma_addr_t dma_addr;
dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len,
queue, OTX2_AF_XDP_FRAME);
if (!err) {
netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);
break;
}
work_done++;
}
if (work_done)
xsk_tx_release(pool);
}

24
drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h Normal file
View File

@@ -0,0 +1,24 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Marvell RVU PF/VF Netdev Devlink
*
* Copyright (C) 2024 Marvell.
*
*/
#ifndef OTX2_XSK_H
#define OTX2_XSK_H
struct otx2_nic;
struct xsk_buff_pool;
int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid);
int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid);
int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma, int idx);
int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
int queue, int budget);
void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx);
#endif /* OTX2_XSK_H */

2
drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c
View File

@@ -82,7 +82,7 @@ static int otx2_qos_sq_aura_pool_init(struct otx2_nic *pfvf, int qidx)
}
for (ptr = 0; ptr < num_sqbs; ptr++) {
err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err)
goto sqb_free;
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);