From 9ca66afe73da16cd2c529e6770cc5aea0c4454df Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:41 -0700 Subject: [PATCH 01/16] xsk: Expose min chunk size to drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drivers should be aware of the range of valid UMEM chunk sizes to be able to allocate their internal structures of an appropriate size. It will be used by mlx5e in the following patches. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan CC: "Björn Töpel" CC: Magnus Karlsson CC: Maciej Fijalkowski Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- include/net/xdp_sock_drv.h | 3 +++ net/xdp/xdp_umem.c | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 0e58c38ce0c1..6406faa3d57d 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -9,6 +9,9 @@ #include #include +#define XDP_UMEM_MIN_CHUNK_SHIFT 11 +#define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT) + #ifdef CONFIG_XDP_SOCKETS void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 869b9b9b9fad..4681e8e8ad94 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -19,8 +19,6 @@ #include "xdp_umem.h" #include "xsk_queue.h" -#define XDP_UMEM_MIN_CHUNK_SIZE 2048 - static DEFINE_IDA(umem_ida); static void xdp_umem_unpin_pages(struct xdp_umem *umem) From e5a3cc83d54019a90119ce0cf17b5e21729b79bf Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:42 -0700 Subject: [PATCH 02/16] net/mlx5e: Use runtime page_shift for striding RQ This commit allows striding RQ to determine MTT page size at runtime, instead of sticking to the compile-time PAGE_SIZE. This functionality will be used by a following commit that adjusts the MTT page size to the XSK frame size. Stick with PAGE_SIZE for XSK on legacy RQ, as frag_stride is not used in data path, it only helps calculate how pages are partitioned into fragments, and PAGE_SIZE will ensure each fragment starts at the beginning of a new allocation unit (XSK frame). Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 41 ++-- .../ethernet/mellanox/mlx5/core/en/params.c | 200 +++++++++++++++--- .../ethernet/mellanox/mlx5/core/en/params.h | 15 +- .../mellanox/mlx5/core/en/xsk/setup.c | 13 +- .../ethernet/mellanox/mlx5/core/en_ethtool.c | 8 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 55 ++--- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 4 +- include/linux/mlx5/qp.h | 2 + 8 files changed, 243 insertions(+), 95 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 9ff746a09a17..7e56a45d0c24 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -93,28 +93,28 @@ struct page_pool; #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \ MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD)) -#define MLX5_MPWRQ_LOG_WQE_SZ 18 -#define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \ - MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0) -#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER) +#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 + +/* Keep in sync with mlx5e_mpwrq_log_wqe_sz. + * These are theoretical maximums, which can be further restricted by + * capabilities. These values are used for static resource allocations and + * sanity checks. + * MLX5_SEND_WQE_MAX_SIZE is a bit bigger than the maximum cacheline-aligned WQE + * size actually used at runtime, but it's not a problem when calculating static + * array sizes. + */ +#define MLX5_UMR_MAX_MTT_SPACE \ + (ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \ + MLX5_UMR_MTT_ALIGNMENT)) +#define MLX5_MPWRQ_MAX_PAGES_PER_WQE \ + rounddown_pow_of_two(MLX5_UMR_MAX_MTT_SPACE / sizeof(struct mlx5_mtt)) #define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8)) #define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2) #define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts))) -/* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between - * WQEs, This page will absorb write overflow by the hardware, when - * receiving packets larger than MTU. These oversize packets are - * dropped by the driver at a later stage. - */ -#define MLX5E_REQUIRED_WQE_MTTS (MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1)) #define MLX5E_MAX_RQ_NUM_MTTS \ (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */ #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024)) -#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \ - (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS)) -#define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \ - (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \ - (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU)) #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM)) #define MLX5E_LOG_MAX_RX_WQE_BULK \ @@ -126,8 +126,7 @@ struct page_pool; #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK) #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa -#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \ - MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW) +#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2 @@ -613,7 +612,7 @@ struct mlx5e_wqe_frag_info { struct mlx5e_mpw_info { u16 consumed_strides; - DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE); + DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE); struct mlx5e_dma_info dma_info[]; }; @@ -622,8 +621,8 @@ struct mlx5e_mpw_info { /* a single cache unit is capable to serve one napi call (for non-striding rq) * or a MPWQE (for striding rq). */ -#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ - MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT) +#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_MAX_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ + MLX5_MPWRQ_MAX_PAGES_PER_WQE : NAPI_POLL_WEIGHT) #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT)) struct mlx5e_page_cache { u32 head; @@ -1008,7 +1007,7 @@ struct mlx5e_profile { void mlx5e_build_ptys2ethtool_map(void); -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev); +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift); void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close); void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 5f8912e8404d..1a0d8f9102df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -7,15 +7,96 @@ #include "en_accel/en_accel.h" #include "en_accel/ipsec.h" -u16 mlx5e_mpwrq_umr_wqe_sz(u8 pages_per_wqe) +static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) { - return sizeof(struct mlx5e_umr_wqe) + - ALIGN(pages_per_wqe * sizeof(struct mlx5_mtt), MLX5_UMR_MTT_ALIGNMENT); + u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size); + + return min_page_shift ? : 12; } -u8 mlx5e_mpwrq_umr_wqebbs(u8 pages_per_wqe) +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { - return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(pages_per_wqe), MLX5_SEND_WQE_BB); + u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev); + u8 req_page_shift = PAGE_SHIFT; + + /* Regular RQ uses order-0 pages, the NIC must be able to map them. */ + if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift)) + min_page_shift = req_page_shift; + + return max(req_page_shift, min_page_shift); +} + +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) +{ + u8 max_pages_per_wqe, max_log_mpwqe_size; + u16 max_wqe_size; + + /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ + max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; + max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), + MLX5_UMR_MTT_ALIGNMENT) / sizeof(struct mlx5_mtt); + max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; + + WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); + + return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ); +} + +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) +{ + u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); + u8 pages_per_wqe; + + pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1; + + /* Sanity check for further calculations to succeed. */ + BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64); + if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE)) + return MLX5_MPWRQ_MAX_PAGES_PER_WQE; + + return pages_per_wqe; +} + +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) +{ + u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift); + u16 umr_wqe_sz; + + umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) + + ALIGN(pages_per_wqe * sizeof(struct mlx5_mtt), MLX5_UMR_MTT_ALIGNMENT); + + WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK); + + return umr_wqe_sz; +} + +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift) +{ + return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift), MLX5_SEND_WQE_BB); +} + +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) +{ + /* Add another page as a buffer between WQEs. This page will absorb + * write overflow by the hardware, when receiving packets larger than + * MTU. These oversize packets are dropped by the driver at a later + * stage. + */ + return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift) + 1); +} + +static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift) +{ + u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift); + + return ilog2(MLX5E_MAX_RQ_NUM_MTTS / mtts_per_wqe); +} + +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift) +{ + return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift) + + mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - + MLX5E_ORDER2_MAX_PACKET_MTU; } u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, @@ -52,14 +133,16 @@ static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk) return MLX5_SKB_FRAG_SZ(headroom + hw_mtu); } -static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) +static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + bool mpwqe) { /* XSK frames are mapped as individual pages, because frames may come in * an arbitrary order from random locations in the UMEM. */ if (xsk) - return PAGE_SIZE; + return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE; /* XDP in mlx5e doesn't support multiple packets per page. */ if (params->xdp_prog) @@ -68,15 +151,18 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5e_params *params, return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false)); } -static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params, +static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(params, xsk); + u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_stride_sz); + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - order_base_2(linear_stride_sz); } -bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, +bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) @@ -96,9 +182,10 @@ bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, } static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, - u8 log_stride_sz, u8 log_num_strides) + u8 log_stride_sz, u8 log_num_strides, + u8 page_shift) { - if (log_stride_sz + log_num_strides != MLX5_MPWRQ_LOG_WQE_SZ) + if (log_stride_sz + log_num_strides != mlx5e_mpwrq_log_wqe_sz(mdev, page_shift)) return false; if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE || @@ -118,28 +205,50 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - s8 log_num_strides; + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + u8 log_num_strides; u8 log_stride_sz; + u8 log_wqe_sz; - if (!mlx5e_rx_is_linear_skb(params, xsk)) + if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) return false; - log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(params, xsk)); - log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - log_stride_sz; + log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); + log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); - return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, log_num_strides); + if (log_wqe_sz < log_stride_sz) + return false; + + log_num_strides = log_wqe_sz - log_stride_sz; + + return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, + log_num_strides, page_shift); } -u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk); + u8 log_pkts_per_wqe, page_shift, max_log_rq_size; + + log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); + page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift); /* Numbers are unsigned, don't subtract to avoid underflow. */ if (params->log_rq_mtu_frames < log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; + /* Ethtool's rx_max_pending is calculated for regular RQ, that uses + * pages of PAGE_SIZE. Max length of an XSK RQ might differ if it uses a + * frame size not equal to PAGE_SIZE. + * A stricter condition is checked in mlx5e_mpwrq_validate_xsk, WARN on + * unexpected failure. + */ + if (WARN_ON_ONCE(params->log_rq_mtu_frames > log_pkts_per_wqe + max_log_rq_size)) + return max_log_rq_size; + return params->log_rq_mtu_frames - log_pkts_per_wqe; } @@ -169,7 +278,7 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) - return order_base_2(mlx5e_rx_get_linear_stride_sz(params, xsk)); + return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); } @@ -178,7 +287,9 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - return MLX5_MPWRQ_LOG_WQE_SZ - + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); } @@ -327,7 +438,9 @@ bool slow_pci_heuristic(struct mlx5_core_dev *mdev) int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { - if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL); + + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) return -EOPNOTSUPP; if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) @@ -339,12 +452,26 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + u16 max_mtu_pkts; + + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) return -EOPNOTSUPP; if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) return -EINVAL; + /* Current RQ length is too big for the given frame size, the + * needed number of WQEs exceeds the maximum. + */ + max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, + mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift)); + if (params->log_rq_mtu_frames > max_mtu_pkts) { + mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", + 1 << params->log_rq_mtu_frames, xsk->chunk_size); + return -EINVAL; + } + return 0; } @@ -358,7 +485,7 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? - BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) : + BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) : BIT(params->log_rq_mtu_frames), BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); @@ -385,7 +512,7 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) && !mlx5e_mpwrq_validate_regular(mdev, params) && (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) || - !mlx5e_rx_is_linear_skb(params, NULL))) + !mlx5e_rx_is_linear_skb(mdev, params, NULL))) MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); mlx5e_set_rq_type(mdev, params); mlx5e_init_rq_type_params(mdev, params); @@ -428,10 +555,10 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, int max_mtu; int i; - if (mlx5e_rx_is_linear_skb(params, xsk)) { + if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) { int frag_stride; - frag_stride = mlx5e_rx_get_linear_stride_sz(params, xsk); + frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false); info->arr[0].frag_size = byte_count; info->arr[0].frag_stride = frag_stride; @@ -528,7 +655,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params)); u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(params, xsk)); + int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); int wqe_size = BIT(log_stride_sz) * num_strides; /* +1 is for the case that the pkt_per_rsrv dont consume the reservation @@ -552,7 +679,7 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk); else - log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + + log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) + mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); break; default: /* MLX5_WQ_TYPE_CYCLIC */ @@ -595,9 +722,11 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: { u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, - log_wqe_num_of_strides)) { + log_wqe_num_of_strides, + page_shift)) { mlx5_core_err(mdev, "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n", log_wqe_stride_size, log_wqe_num_of_strides); @@ -608,7 +737,7 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); MLX5_SET(wq, wq, log_wqe_stride_size, log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); - MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); + MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { MLX5_SET(wq, wq, shampo_enable, true); MLX5_SET(wq, wq, log_reservation_size, @@ -797,7 +926,8 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; - wqebbs = mlx5e_mpwrq_umr_wqebbs(MLX5_MPWRQ_PAGES_PER_WQE) * + /* UMR WQEs for the regular RQ. */ + wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, PAGE_SHIFT) * (1 << mlx5e_get_rq_log_wq_sz(rqp->rqc)); /* If XDP program is attached, XSK may be turned on at any time without @@ -866,7 +996,7 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, mlx5e_build_sq_param_common(mdev, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); - param->is_xdp_mb = !mlx5e_rx_is_linear_skb(params, xsk); + param->is_xdp_mb = !mlx5e_rx_is_linear_skb(mdev, params, xsk); mlx5e_build_tx_cq_param(mdev, params, ¶m->cqp); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 2bb9aba57ea0..3e765b5d42bc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -86,8 +86,13 @@ static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile, /* Striding RQ dynamic parameters */ -u16 mlx5e_mpwrq_umr_wqe_sz(u8 pages_per_wqe); -u8 mlx5e_mpwrq_umr_wqebbs(u8 pages_per_wqe); +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift); +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift); /* Parameter calculations */ @@ -106,12 +111,14 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params * u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); -bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, +bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); -u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); u8 mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index c7c25f20ad72..5129b9bf534f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -5,20 +5,19 @@ #include "en/params.h" #include "en/txrx.h" #include "en/health.h" +#include -/* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may - * change unexpectedly, and mlx5e has a minimum valid stride size for striding - * RQ, keep this check in the driver. +/* The limitation of 2048 can be altered, but shouldn't go beyond the minimal + * stride size of striding RQ. */ -#define MLX5E_MIN_XSK_CHUNK_SIZE 2048 +#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE) bool mlx5e_validate_xsk_param(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, struct mlx5_core_dev *mdev) { /* AF_XDP doesn't support frames larger than PAGE_SIZE. */ - if (xsk->chunk_size > PAGE_SIZE || - xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) + if (xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) return false; /* frag_sz is different for regular and XSK RQs, so ensure that linear @@ -28,7 +27,7 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk); default: /* MLX5_WQ_TYPE_CYCLIC */ - return mlx5e_rx_is_linear_skb(params, xsk); + return mlx5e_rx_is_linear_skb(mdev, params, xsk); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 8ae5cff3361e..c28c1d369855 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -311,7 +311,13 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, struct ethtool_ringparam *param, struct kernel_ethtool_ringparam *kernel_param) { - param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; + /* Limitation for regular RQ. XSK RQ may clamp the queue length in + * mlx5e_mpwqe_get_log_rq_size. + */ + u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, PAGE_SHIFT); + + param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, + max_log_mpwrq_pkts); param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 26f1557843a9..733451de1664 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -68,25 +68,24 @@ #include "qos.h" #include "en/trap.h" -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift) { - bool striding_rq_umr, inline_umr; - u16 max_wqebbs; - u16 umr_wqebbs; + u16 umr_wqebbs, max_wqebbs; + bool striding_rq_umr; striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && MLX5_CAP_ETH(mdev, reg_umr_sq); - max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); - umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(MLX5_MPWRQ_PAGES_PER_WQE); - inline_umr = umr_wqebbs <= max_wqebbs; if (!striding_rq_umr) return false; - if (!inline_umr) { - mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%u) exceeds maximum supported (%u).\n", - umr_wqebbs * MLX5_SEND_WQE_BB, - max_wqebbs * MLX5_SEND_WQE_BB); + + umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift); + max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); + /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is + * calculated from mlx5e_get_max_sq_aligned_wqebbs. + */ + if (WARN_ON(umr_wqebbs > max_wqebbs)) return false; - } + return true; } @@ -211,7 +210,7 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; u8 ds_cnt; - ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mpwqe.pages_per_wqe), + ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift), MLX5_SEND_WQE_DS); cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | @@ -591,13 +590,16 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); - rq->mpwqe.page_shift = PAGE_SHIFT; - rq->mpwqe.pages_per_wqe = MLX5_MPWRQ_PAGES_PER_WQE; - rq->mpwqe.umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(rq->mpwqe.pages_per_wqe); - rq->mpwqe.mtts_per_wqe = MLX5E_REQUIRED_WQE_MTTS; + rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + rq->mpwqe.pages_per_wqe = + mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift); + rq->mpwqe.umr_wqebbs = + mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift); + rq->mpwqe.mtts_per_wqe = + mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift); pool_size = rq->mpwqe.pages_per_wqe << - mlx5e_mpwqe_get_log_rq_size(params, xsk); + mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); rq->mpwqe.num_strides = @@ -4030,14 +4032,16 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, return true; } -static bool mlx5e_params_validate_xdp(struct net_device *netdev, struct mlx5e_params *params) +static bool mlx5e_params_validate_xdp(struct net_device *netdev, + struct mlx5_core_dev *mdev, + struct mlx5e_params *params) { bool is_linear; /* No XSK params: AF_XDP can't be enabled yet at the point of setting * the XDP program. */ - is_linear = mlx5e_rx_is_linear_skb(params, NULL); + is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL); if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", @@ -4074,7 +4078,8 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, if (err) goto out; - if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, &new_params)) { + if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, + &new_params)) { err = -EINVAL; goto out; } @@ -4094,8 +4099,8 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL); bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, &new_params, NULL); - u8 sz_old = mlx5e_mpwqe_get_log_rq_size(params, NULL); - u8 sz_new = mlx5e_mpwqe_get_log_rq_size(&new_params, NULL); + u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL); + u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL); /* Always reset in linear mode - hw_mtu is used in data path. * Check that the mode was non-linear and didn't change. @@ -4553,7 +4558,7 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) new_params = priv->channels.params; new_params.xdp_prog = prog; - if (!mlx5e_params_validate_xdp(netdev, &new_params)) + if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params)) return -EINVAL; return 0; @@ -4924,7 +4929,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) if (!!MLX5_CAP_ETH(mdev, lro_cap) && !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && - mlx5e_check_fragmented_striding_rq_cap(mdev)) + mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT)) netdev->vlan_features |= NETIF_F_LRO; netdev->hw_features = netdev->vlan_features; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index e2f360da6437..d6f5d8ed8dd9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -2431,7 +2431,7 @@ int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool default: /* MLX5_WQ_TYPE_CYCLIC */ rq->wqe.skb_from_cqe = xsk ? mlx5e_xsk_skb_from_cqe_linear : - mlx5e_rx_is_linear_skb(params, NULL) ? + mlx5e_rx_is_linear_skb(mdev, params, NULL) ? mlx5e_skb_from_cqe_linear : mlx5e_skb_from_cqe_nonlinear; rq->post_wqes = mlx5e_post_rx_wqes; @@ -2485,7 +2485,7 @@ static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params) { - rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ? + rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(rq->mdev, params, NULL) ? mlx5e_skb_from_cqe_linear : mlx5e_skb_from_cqe_nonlinear; rq->post_wqes = mlx5e_post_rx_wqes; diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index be640c749d5e..afac93f9552c 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -162,6 +162,8 @@ enum { MLX5_SEND_WQE_MAX_WQEBBS = 16, }; +#define MLX5_SEND_WQE_MAX_SIZE (MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQE_BB) + enum { MLX5_WQE_FMR_PERM_LOCAL_READ = 1 << 27, MLX5_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28, From fa5573359aae63d36518e2821f968ff2f5eae02e Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:43 -0700 Subject: [PATCH 03/16] net/mlx5e: xsk: Use XSK frame size as striding RQ page size XSK RQs support striding RQ linear mode, but the stride size is always set to PAGE_SIZE. It may be larger than the XSK frame size, unnecessarily reducing the useful space in a WQE, but more importantly causing UMEM data corruption in certain cases. Normally, stride size bigger than XSK frame size is not a problem if the hardware enforces the MTU. However, traffic between vports skips the hardware MTU check, and oversized packets may be received. If an oversized packet is bigger than the XSK frame but not bigger than the stride, it will cause overwriting of the adjacent UMEM region. If the packet takes more than one stride, they can be recycled for reuse so it's not a problem when the XSK frame size matches the stride size. To reduce the impact of the above issue, attempt to use the MTT page size for striding RQ that matches the XSK frame size, allowing to safely use 2048-byte frames on an up-to-date firmware. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- .../ethernet/mellanox/mlx5/core/en/params.c | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 1a0d8f9102df..593b684d21d5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -6,6 +6,7 @@ #include "en/port.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec.h" +#include static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) { @@ -16,8 +17,8 @@ static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { + u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT; u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev); - u8 req_page_shift = PAGE_SHIFT; /* Regular RQ uses order-0 pages, the NIC must be able to map them. */ if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift)) @@ -933,12 +934,36 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, /* If XDP program is attached, XSK may be turned on at any time without * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of * both regular RQ and XSK RQ. - * Although mlx5e_mpwqe_get_log_rq_size accepts mlx5e_xsk_param, it - * doesn't affect its return value, as long as params->xdp_prog != NULL, - * so we can just multiply by 2. + * + * XSK uses different values of page_shift, and the total number of UMR + * WQEBBs depends on it. This dependency is complex and not monotonic, + * especially taking into consideration that some of the parameters come + * from capabilities. Hence, we have to try all valid values of XSK + * frame size (and page_shift) to find the maximum. */ - if (params->xdp_prog) - wqebbs *= 2; + if (params->xdp_prog) { + u32 max_xsk_wqebbs = 0; + u8 frame_shift; + + for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; + frame_shift <= PAGE_SHIFT; frame_shift++) { + /* The headroom doesn't affect the calculation. */ + struct mlx5e_xsk_param xsk = { + .chunk_size = 1 << frame_shift, + }; + u32 xsk_wqebbs; + u8 page_shift; + + page_shift = mlx5e_mpwrq_page_shift(mdev, &xsk); + xsk_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift) * + (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, &xsk)); + + if (xsk_wqebbs > max_xsk_wqebbs) + max_xsk_wqebbs = xsk_wqebbs; + } + + wqebbs += max_xsk_wqebbs; + } if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp); From ecc7ad2eab35f180588a60adc5c2b76262dc8d71 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:44 -0700 Subject: [PATCH 04/16] net/mlx5e: Keep a separate MKey for striding RQ Currently, rq->mkey_be keeps a big-endian value of either the PA MKey (for legacy RQ, no address translation) or MTT MKey (for striding RQ, direct address translation). Striding RQ stores the same value in rq->umr_mkey in the native endianness. The next commit will make striding RQ use KSM MKey (indirect address translation) for the unaligned mode of XSK, which will require storing both KSM MKey and PA MKey in the RQ struct. This commit optimizes fields of mlx5e_rq: umr_mkey is removed (it's redundant), mkey_be always points to the PA MKey, and mpwqe.umr_mkey_be points to the MTT MKey (or to the KSM MKey, starting from the next commit). Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 21 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 7e56a45d0c24..308ef06df0d5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -696,6 +696,7 @@ struct mlx5e_rq { struct mlx5e_umr_wqe umr_wqe; struct mlx5e_mpw_info *info; mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq; + __be32 umr_mkey_be; u16 num_strides; u16 actual_wq_head; u8 log_stride_sz; @@ -757,7 +758,6 @@ struct mlx5e_rq { u32 rqn; struct mlx5_core_dev *mdev; struct mlx5e_channel *channel; - u32 umr_mkey; struct mlx5e_dma_info wqe_overflow; /* XDP read-mostly */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 733451de1664..a4edbb85706e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -215,7 +215,7 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); - cseg->umr_mkey = rq->mkey_be; + cseg->umr_mkey = rq->mpwqe.umr_mkey_be; ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; ucseg->xlt_octowords = @@ -365,9 +365,13 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) { u64 num_mtts = mlx5_wq_ll_get_size(&rq->mpwqe.wq) * rq->mpwqe.mtts_per_wqe; + u32 umr_mkey; + int err; - return mlx5e_create_umr_mtt_mkey(mdev, num_mtts, rq->mpwqe.page_shift, - &rq->umr_mkey, rq->wqe_overflow.addr); + err = mlx5e_create_umr_mtt_mkey(mdev, num_mtts, rq->mpwqe.page_shift, + &umr_mkey, rq->wqe_overflow.addr); + rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); + return err; } static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, @@ -575,6 +579,8 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); pool_size = 1 << params->log_rq_mtu_frames; + rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); + switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, @@ -611,7 +617,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, err = mlx5e_create_rq_umr_mkey(mdev, rq); if (err) goto err_rq_drop_page; - rq->mkey_be = cpu_to_be32(rq->umr_mkey); err = mlx5e_rq_alloc_mpwqe_info(rq, node); if (err) @@ -647,8 +652,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, err = mlx5e_init_di_list(rq, wq_sz, node); if (err) goto err_rq_frags; - - rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); } if (xsk) { @@ -695,7 +698,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); wqe->data[0].byte_count = cpu_to_be32(byte_count); - wqe->data[0].lkey = rq->mkey_be; + wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; } else { struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); @@ -740,7 +743,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: kvfree(rq->mpwqe.info); err_rq_mkey: - mlx5_core_destroy_mkey(mdev, rq->umr_mkey); + mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); err_rq_drop_page: mlx5e_free_mpwqe_rq_drop_page(rq); break; @@ -773,7 +776,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: kvfree(rq->mpwqe.info); - mlx5_core_destroy_mkey(rq->mdev, rq->umr_mkey); + mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); mlx5e_free_mpwqe_rq_drop_page(rq); mlx5e_rq_free_shampo(rq); break; From c4418f34955440fbdebd0b00771bdd76e2cd3f63 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:45 -0700 Subject: [PATCH 05/16] net/mlx5: Add MLX5_FLEXIBLE_INLEN to safely calculate cmd inlen Some commands use a flexible array after a common header. Add a macro to safely calculate the total input length of the command, detecting overflows and printing errors with specific values when such overflows happen. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- .../ethernet/mellanox/mlx5/core/mlx5_core.h | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index ad61b86d5769..a806e3de7b7c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -143,6 +143,36 @@ enum mlx5_semaphore_space_address { #define MLX5_DEFAULT_PROF 2 +static inline int mlx5_flexible_inlen(struct mlx5_core_dev *dev, size_t fixed, + size_t item_size, size_t num_items, + const char *func, int line) +{ + int inlen; + + if (fixed > INT_MAX || item_size > INT_MAX || num_items > INT_MAX) { + mlx5_core_err(dev, "%s: %s:%d: input values too big: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + if (check_mul_overflow((int)item_size, (int)num_items, &inlen)) { + mlx5_core_err(dev, "%s: %s:%d: multiplication overflow: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + if (check_add_overflow((int)fixed, inlen, &inlen)) { + mlx5_core_err(dev, "%s: %s:%d: addition overflow: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + return inlen; +} + +#define MLX5_FLEXIBLE_INLEN(dev, fixed, item_size, num_items) \ + mlx5_flexible_inlen(dev, fixed, item_size, num_items, __func__, __LINE__) + int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_cmd_init(struct mlx5_core_dev *dev); From 6470d2e7e8ed8e9dd560d8dc3e09d1100a17ee26 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:46 -0700 Subject: [PATCH 06/16] net/mlx5e: xsk: Use KSM for unaligned XSK UMR MTTs used in striding RQ have certain alignment requirements. While it's guaranteed to work when UMR pages are aligned to the UMR page size, in practice it works then UMR pages are aligned to 8 bytes. However, it's still not enough flexibility for the unaligned mode of XSK. This patch leverages KSM to map UMR pages without alignment requirements, when unaligned XSK is active. The downside is that KSM entries are twice as big as MTTs, which limits the maximum WQE size, so regular RQs and aligned XSK continue using MTTs. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 7 +- .../ethernet/mellanox/mlx5/core/en/params.c | 122 +++++++++++------- .../ethernet/mellanox/mlx5/core/en/params.h | 14 +- .../ethernet/mellanox/mlx5/core/en/xsk/pool.c | 1 + .../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 14 -- .../ethernet/mellanox/mlx5/core/en_ethtool.c | 2 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 91 +++++++++---- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 34 +++-- include/linux/mlx5/qp.h | 6 + 9 files changed, 191 insertions(+), 100 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 308ef06df0d5..449c016262f4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -112,8 +112,10 @@ struct page_pool; #define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8)) #define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2) #define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts))) +#define MLX5_KSM_OCTW(ksms) (ksms) #define MLX5E_MAX_RQ_NUM_MTTS \ (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */ +#define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */ #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024)) #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM)) @@ -265,6 +267,7 @@ struct mlx5e_umr_wqe { union { DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts); DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms); + DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms); }; }; @@ -708,6 +711,7 @@ struct mlx5e_rq { u8 pages_per_wqe; u8 umr_wqebbs; u8 mtts_per_wqe; + u8 unaligned; struct mlx5e_shampo_hd *shampo; } mpwqe; }; @@ -1007,7 +1011,8 @@ struct mlx5e_profile { void mlx5e_build_ptys2ethtool_map(void); -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift); +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, + bool unaligned); void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close); void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 593b684d21d5..0f18031a871a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -27,15 +27,16 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs return max(req_page_shift, min_page_shift); } -u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) { + u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); u8 max_pages_per_wqe, max_log_mpwqe_size; u16 max_wqe_size; /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), - MLX5_UMR_MTT_ALIGNMENT) / sizeof(struct mlx5_mtt); + MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size; max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); @@ -43,9 +44,9 @@ u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ); } -u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) { - u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); + u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned); u8 pages_per_wqe; pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1; @@ -58,45 +59,58 @@ u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) return pages_per_wqe; } -u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift) +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) { - u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift); + u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); + u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned); u16 umr_wqe_sz; umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) + - ALIGN(pages_per_wqe * sizeof(struct mlx5_mtt), MLX5_UMR_MTT_ALIGNMENT); + ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT); WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK); return umr_wqe_sz; } -u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift) +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) { - return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift), MLX5_SEND_WQE_BB); + return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, unaligned), + MLX5_SEND_WQE_BB); } -u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift) +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) { /* Add another page as a buffer between WQEs. This page will absorb * write overflow by the hardware, when receiving packets larger than * MTU. These oversize packets are dropped by the driver at a later * stage. */ - return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift) + 1); + return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned) + 1); } -static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift) +u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned) { - u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift); + if (unaligned) + return min(MLX5E_MAX_RQ_NUM_KSMS, + 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size)); - return ilog2(MLX5E_MAX_RQ_NUM_MTTS / mtts_per_wqe); + return MLX5E_MAX_RQ_NUM_MTTS; } -u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift) +static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift, + bool unaligned) { - return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift) + - mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - + u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, unaligned); + u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, unaligned); + + return ilog2(max_entries / mtts_per_wqe); +} + +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned) +{ + return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned) + + mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) - MLX5E_ORDER2_MAX_PACKET_MTU; } @@ -158,8 +172,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, { u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; - return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - order_base_2(linear_stride_sz); + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) - + order_base_2(linear_stride_sz); } bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, @@ -184,9 +200,10 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, u8 log_stride_sz, u8 log_num_strides, - u8 page_shift) + u8 page_shift, bool unaligned) { - if (log_stride_sz + log_num_strides != mlx5e_mpwrq_log_wqe_sz(mdev, page_shift)) + if (log_stride_sz + log_num_strides != + mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned)) return false; if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE || @@ -207,6 +224,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; u8 log_num_strides; u8 log_stride_sz; u8 log_wqe_sz; @@ -215,7 +233,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, return false; log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); - log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift); + log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned); if (log_wqe_sz < log_stride_sz) return false; @@ -223,7 +241,8 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, log_num_strides = log_wqe_sz - log_stride_sz; return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, - log_num_strides, page_shift); + log_num_strides, page_shift, + unaligned); } u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, @@ -231,10 +250,11 @@ u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { u8 log_pkts_per_wqe, page_shift, max_log_rq_size; + bool unaligned = xsk ? xsk->unaligned : false; log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift); + max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned); /* Numbers are unsigned, don't subtract to avoid underflow. */ if (params->log_rq_mtu_frames < @@ -289,8 +309,9 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; - return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift) - + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) - mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); } @@ -441,7 +462,7 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params { u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL); - if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, false)) return -EOPNOTSUPP; if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) @@ -454,9 +475,10 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa struct mlx5e_xsk_param *xsk) { u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; u16 max_mtu_pkts; - if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift)) + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, xsk->unaligned)) return -EOPNOTSUPP; if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) @@ -466,7 +488,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa * needed number of WQEs exceeds the maximum. */ max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, - mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift)); + mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, unaligned)); if (params->log_rq_mtu_frames > max_mtu_pkts) { mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", 1 << params->log_rq_mtu_frames, xsk->chunk_size); @@ -724,13 +746,15 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, log_wqe_num_of_strides, - page_shift)) { + page_shift, unaligned)) { mlx5_core_err(mdev, - "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n", - log_wqe_stride_size, log_wqe_num_of_strides); + "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, unaligned %d\n", + log_wqe_stride_size, log_wqe_num_of_strides, + unaligned); return -EINVAL; } @@ -850,13 +874,6 @@ static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev, param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; } -static u8 mlx5e_get_rq_log_wq_sz(void *rqc) -{ - void *wq = MLX5_ADDR_OF(rqc, rqc, wq); - - return MLX5_GET(wq, wq, log_wq_sz); -} - /* This function calculates the maximum number of headers entries that are needed * per WQE, the formula is based on the size of the reservations and the * restriction we have about max packets for reservation that is equal to max @@ -917,6 +934,19 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, return wqebbs; } +static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk) +{ + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; + u8 umr_wqebbs; + + umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, unaligned); + + return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); +} + static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rqp) @@ -928,8 +958,7 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; /* UMR WQEs for the regular RQ. */ - wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, PAGE_SHIFT) * - (1 << mlx5e_get_rq_log_wq_sz(rqp->rqc)); + wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); /* If XDP program is attached, XSK may be turned on at any time without * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of @@ -950,16 +979,17 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, /* The headroom doesn't affect the calculation. */ struct mlx5e_xsk_param xsk = { .chunk_size = 1 << frame_shift, + .unaligned = false, }; - u32 xsk_wqebbs; - u8 page_shift; - page_shift = mlx5e_mpwrq_page_shift(mdev, &xsk); - xsk_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift) * - (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, &xsk)); + /* XSK aligned mode. */ + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - if (xsk_wqebbs > max_xsk_wqebbs) - max_xsk_wqebbs = xsk_wqebbs; + /* XSK unaligned mode, frame size is a power of two. */ + xsk.unaligned = true; + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); } wqebbs += max_xsk_wqebbs; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 3e765b5d42bc..cb862c478376 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -9,6 +9,7 @@ struct mlx5e_xsk_param { u16 headroom; u16 chunk_size; + bool unaligned; }; struct mlx5e_cq_param { @@ -87,12 +88,13 @@ static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile, /* Striding RQ dynamic parameters */ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); -u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); -u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); -u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift); -u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift); -u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift); -u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift); +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); +u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned); +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned); /* Parameter calculations */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index 2c520394aa1d..6058b1e72c6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -72,6 +72,7 @@ void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *x { xsk->headroom = xsk_pool_get_headroom(pool); xsk->chunk_size = xsk_pool_get_chunk_size(pool); + xsk->unaligned = pool->unaligned; } static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index cc18d97d8ee0..a8cfab4a393c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -7,8 +7,6 @@ #include "en.h" #include -#define MLX5E_MTT_PTAG_MASK 0xfffffffffffffff8ULL - /* RX data path */ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, @@ -23,7 +21,6 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info) { -retry: dma_info->xsk = xsk_buff_alloc(rq->xsk_pool); if (!dma_info->xsk) return -ENOMEM; @@ -35,17 +32,6 @@ static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, */ dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk); - /* MTT page mapping has alignment requirements. If they are not - * satisfied, leak the descriptor so that it won't come again, and try - * to allocate a new one. - */ - if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { - if (unlikely(dma_info->addr & ~MLX5E_MTT_PTAG_MASK)) { - xsk_buff_discard(dma_info->xsk); - goto retry; - } - } - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index c28c1d369855..26f1ac4683e7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -314,7 +314,7 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, /* Limitation for regular RQ. XSK RQ may clamp the queue length in * mlx5e_mpwqe_get_log_rq_size. */ - u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, PAGE_SHIFT); + u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, PAGE_SHIFT, false); param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, max_log_mpwrq_pkts); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index a4edbb85706e..fbbc2e792c27 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -68,7 +68,8 @@ #include "qos.h" #include "en/trap.h" -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift) +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, + bool unaligned) { u16 umr_wqebbs, max_wqebbs; bool striding_rq_umr; @@ -78,7 +79,7 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_ if (!striding_rq_umr) return false; - umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift); + umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, unaligned); max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is * calculated from mlx5e_get_max_sq_aligned_wqebbs. @@ -208,9 +209,11 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, { struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; + u16 octowords; u8 ds_cnt; - ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift), + ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, + rq->mpwqe.unaligned), MLX5_SEND_WQE_DS); cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | @@ -218,8 +221,9 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, cseg->umr_mkey = rq->mpwqe.umr_mkey_be; ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; - ucseg->xlt_octowords = - cpu_to_be16(MLX5_MTT_OCTW(rq->mpwqe.pages_per_wqe)); + octowords = rq->mpwqe.unaligned ? MLX5_KSM_OCTW(rq->mpwqe.pages_per_wqe) : + MLX5_MTT_OCTW(rq->mpwqe.pages_per_wqe); + ucseg->xlt_octowords = cpu_to_be16(octowords); ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); } @@ -279,39 +283,51 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) return 0; } -static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, - u64 npages, u8 page_shift, u32 *umr_mkey, - dma_addr_t filler_addr) +static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, + u32 npages, u8 page_shift, u32 *umr_mkey, + dma_addr_t filler_addr, bool unaligned) { struct mlx5_mtt *mtt; + struct mlx5_ksm *ksm; + u32 octwords; int inlen; void *mkc; u32 *in; int err; int i; - inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages; + if (unaligned && !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { + mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); + return -EINVAL; + } + + inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), + unaligned ? sizeof(*ksm) : sizeof(*mtt), + npages); + if (inlen < 0) + return inlen; in = kvzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; + octwords = unaligned ? MLX5_KSM_OCTW(npages) : MLX5_MTT_OCTW(npages); + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); + MLX5_SET(mkc, mkc, access_mode_1_0, + unaligned ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT); mlx5e_mkey_set_relaxed_ordering(mdev, mkc); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); MLX5_SET64(mkc, mkc, len, npages << page_shift); - MLX5_SET(mkc, mkc, translations_octword_size, - MLX5_MTT_OCTW(npages)); + MLX5_SET(mkc, mkc, translations_octword_size, octwords); MLX5_SET(mkc, mkc, log_page_size, page_shift); - MLX5_SET(create_mkey_in, in, translations_octword_actual_size, - MLX5_MTT_OCTW(npages)); + MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); /* Initialize the mkey with all MTTs pointing to a default * page (filler_addr). When the channels are activated, UMR @@ -319,9 +335,20 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, * the RQ's pool, while the gaps (wqe_overflow) remain mapped * to the default page. */ - mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); - for (i = 0 ; i < npages ; i++) - mtt[i].ptag = cpu_to_be64(filler_addr); + if (unaligned) { + ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages; i++) + ksm[i] = (struct mlx5_ksm) { + .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), + .va = cpu_to_be64(filler_addr), + }; + } else { + mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages; i++) + mtt[i] = (struct mlx5_mtt) { + .ptag = cpu_to_be64(filler_addr), + }; + } err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); @@ -364,12 +391,24 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) { - u64 num_mtts = mlx5_wq_ll_get_size(&rq->mpwqe.wq) * rq->mpwqe.mtts_per_wqe; + u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); + u32 num_entries, max_num_entries; u32 umr_mkey; int err; - err = mlx5e_create_umr_mtt_mkey(mdev, num_mtts, rq->mpwqe.page_shift, - &umr_mkey, rq->wqe_overflow.addr); + max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.unaligned); + + /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ + if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, + &num_entries) || + num_entries > max_num_entries)) + mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", + __func__, wq_size, rq->mpwqe.mtts_per_wqe, + max_num_entries); + + err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, + &umr_mkey, rq->wqe_overflow.addr, + rq->mpwqe.unaligned); rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); return err; } @@ -597,12 +636,16 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + rq->mpwqe.unaligned = xsk ? xsk->unaligned : false; rq->mpwqe.pages_per_wqe = - mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift); + mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, + rq->mpwqe.unaligned); rq->mpwqe.umr_wqebbs = - mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift); + mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, + rq->mpwqe.unaligned); rq->mpwqe.mtts_per_wqe = - mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift); + mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, + rq->mpwqe.unaligned); pool_size = rq->mpwqe.pages_per_wqe << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); @@ -4932,7 +4975,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) if (!!MLX5_CAP_ETH(mdev, lro_cap) && !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && - mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT)) + mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, false)) netdev->vlan_features |= NETIF_F_LRO; netdev->hw_features = netdev->vlan_features; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index d6f5d8ed8dd9..de929fde8cc6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -673,6 +673,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) struct mlx5e_icosq *sq = rq->icosq; struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5e_umr_wqe *umr_wqe; + u32 offset; /* 17-bit value with MTT. */ u16 pi; int err; int i; @@ -694,13 +695,27 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); - memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts)); + memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); - for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { - err = mlx5e_page_alloc(rq, dma_info); - if (unlikely(err)) - goto err_unmap; - umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR); + if (unlikely(rq->mpwqe.unaligned)) { + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { + err = mlx5e_page_alloc(rq, dma_info); + if (unlikely(err)) + goto err_unmap; + umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(dma_info->addr), + }; + } + } else { + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { + err = mlx5e_page_alloc(rq, dma_info); + if (unlikely(err)) + goto err_unmap; + umr_wqe->inline_mtts[i] = (struct mlx5_mtt) { + .ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR), + }; + } } bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); @@ -709,8 +724,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) umr_wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR); - umr_wqe->uctrl.xlt_offset = - cpu_to_be16(MLX5_ALIGNED_MTTS_OCTW(ix * rq->mpwqe.mtts_per_wqe)); + + offset = ix * rq->mpwqe.mtts_per_wqe; + if (!rq->mpwqe.unaligned) + offset = MLX5_ALIGNED_MTTS_OCTW(offset); + umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset); sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX, diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index afac93f9552c..4657d5c54abe 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -478,6 +478,12 @@ struct mlx5_klm { __be64 va; }; +struct mlx5_ksm { + __be32 reserved; + __be32 key; + __be64 va; +}; + struct mlx5_stride_block_entry { __be16 stride; __be16 bcount; From f2f1675836015e79ba9720d4f7f02441fd0bb5e5 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:47 -0700 Subject: [PATCH 07/16] xsk: Remove unused xsk_buff_discard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit removed the last usage of xsk_buff_discard in mlx5e, so the function that is no longer used can be removed. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan CC: "Björn Töpel" CC: Magnus Karlsson CC: Maciej Fijalkowski Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- include/net/xdp_sock_drv.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 6406faa3d57d..9c0d860609ba 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -107,13 +107,6 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) xp_free(xskb); } -static inline void xsk_buff_discard(struct xdp_buff *xdp) -{ - struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - - xp_release(xskb); -} - static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) { xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; From 0b9c86c78586c209926fc360687dd4a13666840b Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:48 -0700 Subject: [PATCH 08/16] net/mlx5e: Fix calculations for ICOSQ size WQEs must not cross page boundaries, they are padded with NOPs if they don't fit the page. mlx5e_mpwrq_total_umr_wqebbs doesn't take into account this padding, risking reserving not enough space. The padding is not straightforward to add to this calculation, because WQEs of different sizes may be mixed together in the queue. If each page ends with a big WQE that doesn't fit and requires at most its size minus 1 WQEBB of padding, the total space can be much bigger than in case when smaller WQEs take advantage of this padding. Replace the wrong exact calculation by the following estimation. Each padding can be at most the size of the maximum WQE used in the queue minus one WQEBB. Let's call the rest of the page "useful space". If we divide the total size of all needed WQEs by this useful space, rounding up, we'll get the number of pages, which is enough to contain all these WQEs. It's correct, because every WQE that appeared on the boundary between two blocks of useful space would start in the useful space of one page and end in the padding of the same page, while our estimation reserved space for its tail in the next space, making the estimation not smaller than the real space occupied in the queue. The code actually uses a looser estimation: instead of taking the maximum size of all used WQE types minus 1 WQEBB, it takes the maximum hardware size of a WQE. It's made for simplicity and extensibility. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- .../net/ethernet/mellanox/mlx5/core/en/params.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 0f18031a871a..68bc66cbd8a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -951,7 +951,7 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rqp) { - u32 wqebbs; + u32 wqebbs, total_pages, useful_space; /* MLX5_WQ_TYPE_CYCLIC */ if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) @@ -998,6 +998,18 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp); + /* UMR WQEs don't cross the page boundary, they are padded with NOPs. + * This padding is always smaller than the max WQE size. That gives us + * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes + * per page. The number of pages is estimated as the total size of WQEs + * divided by the useful space in page, rounding up. If some WQEs don't + * fully fit into the useful space, they can occupy part of the padding, + * which proves this estimation to be correct (reserve enough space). + */ + useful_space = PAGE_SIZE - mlx5e_get_max_sq_wqebbs(mdev) + MLX5_SEND_WQE_BB; + total_pages = DIV_ROUND_UP(wqebbs * MLX5_SEND_WQE_BB, useful_space); + wqebbs = total_pages * (PAGE_SIZE / MLX5_SEND_WQE_BB); + return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs)); } From 707f908e31d7e2c3b24fe8cafa773f67e44fc5ef Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:49 -0700 Subject: [PATCH 09/16] net/mlx5e: Optimize the page cache reducing its size 2x RX page cache stores dma_info structs, that consist of a pointer to struct page and a DMA address. In fact, the DMA address is extracted from struct page using page_pool_get_dma_addr when a page is pushed to the cache. By moving this call to the point when a page is popped from the cache, we can avoid storing the DMA address in the cache, effectively reducing its size by two times without losing any functionality. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 4 +--- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 8 ++++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 449c016262f4..6b91fa7f2221 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -630,7 +630,7 @@ struct mlx5e_mpw_info { struct mlx5e_page_cache { u32 head; u32 tail; - struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE]; + struct page *page_cache[MLX5E_CACHE_SIZE]; }; struct mlx5e_rq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index fbbc2e792c27..b1d8fd08887b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -830,13 +830,11 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) for (i = rq->page_cache.head; i != rq->page_cache.tail; i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { - struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i]; - /* With AF_XDP, page_cache is not used, so this loop is not * entered, and it's safe to call mlx5e_page_release_dynamic * directly. */ - mlx5e_page_release_dynamic(rq, dma_info->page, false); + mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false); } xdp_rxq_info_unreg(&rq->xdp_rxq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index de929fde8cc6..b8aa6f843675 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -245,8 +245,7 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) return false; } - cache->page_cache[cache->tail].page = page; - cache->page_cache[cache->tail].addr = page_pool_get_dma_addr(page); + cache->page_cache[cache->tail] = page; cache->tail = tail_next; return true; } @@ -262,12 +261,13 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, return false; } - if (page_ref_count(cache->page_cache[cache->head].page) != 1) { + if (page_ref_count(cache->page_cache[cache->head]) != 1) { stats->cache_busy++; return false; } - *dma_info = cache->page_cache[cache->head]; + dma_info->page = cache->page_cache[cache->head]; + dma_info->addr = page_pool_get_dma_addr(dma_info->page); cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1); stats->cache_reuse++; From 79008676d533b7ea92a5938b6526411ca5f2657f Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:50 -0700 Subject: [PATCH 10/16] net/mlx5e: Rename mlx5e_dma_info to prepare for removal of DMA address The next commit will remove the DMA address from the struct currently called mlx5e_dma_info, because the same value can be retrieved with page_pool_get_dma_addr(page) in almost all cases, with the notable exception of SHAMPO (HW GRO implementation) that modifies this address on the fly, after the initial allocation. To keep the SHAMPO logic intact, struct mlx5e_dma_info remains in the SHAMPO code, consisting of addr and page (XSK is not compatible with SHAMPO). The struct used in all other places is renamed to mlx5e_alloc_unit, allowing the next commit to remove the addr field without affecting SHAMPO. The new name means "allocation unit", and it's more appropriate after the field with the DMA address gets removed. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 15 +- .../ethernet/mellanox/mlx5/core/en/xsk/rx.c | 4 +- .../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 8 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 17 +- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 190 +++++++++--------- 5 files changed, 123 insertions(+), 111 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 6b91fa7f2221..06f49ef71b5b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -474,7 +474,7 @@ struct mlx5e_txqsq { cqe_ts_to_ns ptp_cyc2time; } ____cacheline_aligned_in_smp; -struct mlx5e_dma_info { +struct mlx5e_alloc_unit { dma_addr_t addr; union { struct page *page; @@ -608,7 +608,7 @@ struct mlx5e_icosq { } ____cacheline_aligned_in_smp; struct mlx5e_wqe_frag_info { - struct mlx5e_dma_info *di; + struct mlx5e_alloc_unit *au; u32 offset; bool last_in_page; }; @@ -616,7 +616,7 @@ struct mlx5e_wqe_frag_info { struct mlx5e_mpw_info { u16 consumed_strides; DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE); - struct mlx5e_dma_info dma_info[]; + struct mlx5e_alloc_unit alloc_units[]; }; #define MLX5E_MAX_RX_FRAGS 4 @@ -665,6 +665,11 @@ struct mlx5e_rq_frags_info { u8 wqe_bulk; }; +struct mlx5e_dma_info { + dma_addr_t addr; + struct page *page; +}; + struct mlx5e_shampo_hd { u32 mkey; struct mlx5e_dma_info *info; @@ -690,7 +695,7 @@ struct mlx5e_rq { struct { struct mlx5_wq_cyc wq; struct mlx5e_wqe_frag_info *frags; - struct mlx5e_dma_info *di; + struct mlx5e_alloc_unit *alloc_units; struct mlx5e_rq_frags_info info; mlx5e_fp_skb_from_cqe skb_from_cqe; } wqe; @@ -1142,8 +1147,6 @@ void mlx5e_destroy_q_counters(struct mlx5e_priv *priv); int mlx5e_open_drop_rq(struct mlx5e_priv *priv, struct mlx5e_rq *drop_rq); void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq); -int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node); -void mlx5e_free_di_list(struct mlx5e_rq *rq); int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn); void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c index 6245dfde6666..4441d35943d1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c @@ -30,7 +30,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, u32 head_offset, u32 page_idx) { - struct xdp_buff *xdp = wi->dma_info[page_idx].xsk; + struct xdp_buff *xdp = wi->alloc_units[page_idx].xsk; struct bpf_prog *prog; /* Check packet size. Note LRO doesn't use linear SKB */ @@ -83,7 +83,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { - struct xdp_buff *xdp = wi->di->xsk; + struct xdp_buff *xdp = wi->au->xsk; struct bpf_prog *prog; /* wi->offset is not used in this function, because xdp->data and the diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index a8cfab4a393c..c9bdd2c04922 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -19,10 +19,10 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, u32 cqe_bcnt); static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) + struct mlx5e_alloc_unit *au) { - dma_info->xsk = xsk_buff_alloc(rq->xsk_pool); - if (!dma_info->xsk) + au->xsk = xsk_buff_alloc(rq->xsk_pool); + if (!au->xsk) return -ENOMEM; /* Store the DMA address without headroom. In striding RQ case, we just @@ -30,7 +30,7 @@ static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, * when creating a WQE. In non-striding RQ case, headroom is accounted * in mlx5e_alloc_rx_wqe. */ - dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk); + au->addr = xsk_buff_xdp_get_frame_dma(au->xsk); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b1d8fd08887b..47d60a53727f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -271,7 +271,7 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); size_t alloc_size; - alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, dma_info, + alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe)); rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); @@ -433,7 +433,7 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) struct mlx5e_wqe_frag_info *prev = NULL; int i; - next_frag.di = &rq->wqe.di[0]; + next_frag.au = &rq->wqe.alloc_units[0]; for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; @@ -443,7 +443,7 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { - next_frag.di++; + next_frag.au++; next_frag.offset = 0; if (prev) prev->last_in_page = true; @@ -460,12 +460,13 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) prev->last_in_page = true; } -int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) +static int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) { int len = wq_sz << rq->wqe.info.log_num_frags; - rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node); - if (!rq->wqe.di) + rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)), + GFP_KERNEL, node); + if (!rq->wqe.alloc_units) return -ENOMEM; mlx5e_init_frags_partition(rq); @@ -473,9 +474,9 @@ int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) return 0; } -void mlx5e_free_di_list(struct mlx5e_rq *rq) +static void mlx5e_free_di_list(struct mlx5e_rq *rq) { - kvfree(rq->wqe.di); + kvfree(rq->wqe.alloc_units); } static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index b8aa6f843675..83941a4c40a1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -77,7 +77,7 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = { static struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i) { - size_t isz = struct_size(rq->mpwqe.info, dma_info, rq->mpwqe.pages_per_wqe); + size_t isz = struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe); return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz)); } @@ -250,8 +250,7 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) return true; } -static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) +static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) { struct mlx5e_page_cache *cache = &rq->page_cache; struct mlx5e_rq_stats *stats = rq->stats; @@ -266,48 +265,44 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, return false; } - dma_info->page = cache->page_cache[cache->head]; - dma_info->addr = page_pool_get_dma_addr(dma_info->page); + au->page = cache->page_cache[cache->head]; + au->addr = page_pool_get_dma_addr(au->page); cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1); stats->cache_reuse++; - dma_sync_single_for_device(rq->pdev, dma_info->addr, - /* Non-XSK always uses PAGE_SIZE. */ - PAGE_SIZE, - DMA_FROM_DEVICE); + /* Non-XSK always uses PAGE_SIZE. */ + dma_sync_single_for_device(rq->pdev, au->addr, PAGE_SIZE, DMA_FROM_DEVICE); return true; } -static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) +static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) { - if (mlx5e_rx_cache_get(rq, dma_info)) + if (mlx5e_rx_cache_get(rq, au)) return 0; - dma_info->page = page_pool_dev_alloc_pages(rq->page_pool); - if (unlikely(!dma_info->page)) + au->page = page_pool_dev_alloc_pages(rq->page_pool); + if (unlikely(!au->page)) return -ENOMEM; /* Non-XSK always uses PAGE_SIZE. */ - dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE, - rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); - if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) { - page_pool_recycle_direct(rq->page_pool, dma_info->page); - dma_info->page = NULL; + au->addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE, + rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); + if (unlikely(dma_mapping_error(rq->pdev, au->addr))) { + page_pool_recycle_direct(rq->page_pool, au->page); + au->page = NULL; return -ENOMEM; } - page_pool_set_dma_addr(dma_info->page, dma_info->addr); + page_pool_set_dma_addr(au->page, au->addr); return 0; } -static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) +static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) { if (rq->xsk_pool) - return mlx5e_xsk_page_alloc_pool(rq, dma_info); + return mlx5e_xsk_page_alloc_pool(rq, au); else - return mlx5e_page_alloc_pool(rq, dma_info); + return mlx5e_page_alloc_pool(rq, au); } void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page) @@ -335,7 +330,7 @@ void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool rec } static inline void mlx5e_page_release(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info, + struct mlx5e_alloc_unit *au, bool recycle) { if (rq->xsk_pool) @@ -343,9 +338,9 @@ static inline void mlx5e_page_release(struct mlx5e_rq *rq, * put into the Reuse Ring, because there is no way to return * the page to the userspace when the interface goes down. */ - xsk_buff_free(dma_info->xsk); + xsk_buff_free(au->xsk); else - mlx5e_page_release_dynamic(rq, dma_info->page, recycle); + mlx5e_page_release_dynamic(rq, au->page, recycle); } static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, @@ -354,12 +349,12 @@ static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, int err = 0; if (!frag->offset) - /* On first frag (offset == 0), replenish page (dma_info actually). - * Other frags that point to the same dma_info (with a different + /* On first frag (offset == 0), replenish page (alloc_unit actually). + * Other frags that point to the same alloc_unit (with a different * offset) should just use the new one without replenishing again * by themselves. */ - err = mlx5e_page_alloc(rq, frag->di); + err = mlx5e_page_alloc(rq, frag->au); return err; } @@ -369,7 +364,7 @@ static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq, bool recycle) { if (frag->last_in_page) - mlx5e_page_release(rq, frag->di, recycle); + mlx5e_page_release(rq, frag->au, recycle); } static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix) @@ -392,7 +387,7 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, goto free_frags; headroom = i == 0 ? rq->buff.headroom : 0; - wqe->data[i].addr = cpu_to_be64(frag->di->addr + + wqe->data[i].addr = cpu_to_be64(frag->au->addr + frag->offset + headroom); } @@ -458,36 +453,34 @@ static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk) static inline void mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, - struct mlx5e_dma_info *di, u32 frag_offset, u32 len, + struct mlx5e_alloc_unit *au, u32 frag_offset, u32 len, unsigned int truesize) { - dma_sync_single_for_cpu(rq->pdev, - di->addr + frag_offset, + dma_sync_single_for_cpu(rq->pdev, au->addr + frag_offset, len, DMA_FROM_DEVICE); - page_ref_inc(di->page); + page_ref_inc(au->page); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - di->page, frag_offset, len, truesize); + au->page, frag_offset, len, truesize); } static inline void mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb, - struct mlx5e_dma_info *dma_info, + struct page *page, dma_addr_t addr, int offset_from, int dma_offset, u32 headlen) { - const void *from = page_address(dma_info->page) + offset_from; + const void *from = page_address(page) + offset_from; /* Aligning len to sizeof(long) optimizes memcpy performance */ unsigned int len = ALIGN(headlen, sizeof(long)); - dma_sync_single_for_cpu(pdev, dma_info->addr + dma_offset, len, - DMA_FROM_DEVICE); + dma_sync_single_for_cpu(pdev, addr + dma_offset, len, DMA_FROM_DEVICE); skb_copy_to_linear_data(skb, from, len); } static void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle) { + struct mlx5e_alloc_unit *alloc_units = wi->alloc_units; bool no_xdp_xmit; - struct mlx5e_dma_info *dma_info = wi->dma_info; int i; /* A common case for AF_XDP. */ @@ -498,7 +491,7 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle for (i = 0; i < rq->mpwqe.pages_per_wqe; i++) if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap)) - mlx5e_page_release(rq, &dma_info[i], recycle); + mlx5e_page_release(rq, &alloc_units[i], recycle); } static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n) @@ -583,11 +576,13 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) << MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; if (!(header_offset & (PAGE_SIZE - 1))) { - err = mlx5e_page_alloc(rq, dma_info); + struct mlx5e_alloc_unit au; + + err = mlx5e_page_alloc(rq, &au); if (unlikely(err)) goto err_unmap; - addr = dma_info->addr; - page = dma_info->page; + page = dma_info->page = au.page; + addr = dma_info->addr = page_pool_get_dma_addr(au.page); } else { dma_info->addr = addr + header_offset; dma_info->page = page; @@ -619,8 +614,12 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, while (--i >= 0) { dma_info = &shampo->info[--index]; if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) { + struct mlx5e_alloc_unit au = { + .page = dma_info->page, + }; + dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE); - mlx5e_page_release(rq, dma_info, true); + mlx5e_page_release(rq, &au, true); } } rq->stats->buff_alloc_err++; @@ -669,7 +668,7 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); - struct mlx5e_dma_info *dma_info = &wi->dma_info[0]; + struct mlx5e_alloc_unit *au = &wi->alloc_units[0]; struct mlx5e_icosq *sq = rq->icosq; struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5e_umr_wqe *umr_wqe; @@ -698,22 +697,22 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); if (unlikely(rq->mpwqe.unaligned)) { - for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { - err = mlx5e_page_alloc(rq, dma_info); + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) { + err = mlx5e_page_alloc(rq, au); if (unlikely(err)) goto err_unmap; umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { .key = rq->mkey_be, - .va = cpu_to_be64(dma_info->addr), + .va = cpu_to_be64(au->addr), }; } } else { - for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) { - err = mlx5e_page_alloc(rq, dma_info); + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) { + err = mlx5e_page_alloc(rq, au); if (unlikely(err)) goto err_unmap; umr_wqe->inline_mtts[i] = (struct mlx5_mtt) { - .ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR), + .ptag = cpu_to_be64(au->addr | MLX5_EN_WR), }; } } @@ -744,8 +743,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) err_unmap: while (--i >= 0) { - dma_info--; - mlx5e_page_release(rq, dma_info, true); + au--; + mlx5e_page_release(rq, au, true); } err: @@ -778,8 +777,12 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close hd_info = &shampo->info[index]; hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE); if (hd_info->page != deleted_page) { + struct mlx5e_alloc_unit au = { + .page = hd_info->page, + }; + deleted_page = hd_info->page; - mlx5e_page_release(rq, hd_info, false); + mlx5e_page_release(rq, &au, false); } } @@ -1554,7 +1557,7 @@ static struct sk_buff * mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { - struct mlx5e_dma_info *di = wi->di; + struct mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; @@ -1562,11 +1565,11 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, void *va, *data; u32 frag_size; - va = page_address(di->page) + wi->offset; + va = page_address(au->page) + wi->offset; data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset, + dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); @@ -1576,7 +1579,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp); - if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) + if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) return NULL; /* page/packet was consumed by XDP */ rx_headroom = xdp.data - xdp.data_hard_start; @@ -1589,7 +1592,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, return NULL; /* queue up for recycling/reuse */ - page_ref_inc(di->page); + page_ref_inc(au->page); return skb; } @@ -1600,8 +1603,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; struct mlx5e_wqe_frag_info *head_wi = wi; + struct mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; - struct mlx5e_dma_info *di = wi->di; struct skb_shared_info *sinfo; u32 frag_consumed_bytes; struct bpf_prog *prog; @@ -1610,10 +1613,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi u32 truesize; void *va; - va = page_address(di->page) + wi->offset; + va = page_address(au->page) + wi->offset; frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset, + dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset, rq->buff.frame0_sz, DMA_FROM_DEVICE); net_prefetchw(va); /* xdp_frame data area */ net_prefetch(va + rx_headroom); @@ -1629,11 +1632,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi while (cqe_bcnt) { skb_frag_t *frag; - di = wi->di; + au = wi->au; frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_for_cpu(rq->pdev, di->addr + wi->offset, + dma_sync_single_for_cpu(rq->pdev, au->addr + wi->offset, frag_consumed_bytes, DMA_FROM_DEVICE); if (!xdp_buff_has_frags(&xdp)) { @@ -1646,11 +1649,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi } frag = &sinfo->frags[sinfo->nr_frags++]; - __skb_frag_set_page(frag, di->page); + __skb_frag_set_page(frag, au->page); skb_frag_off_set(frag, wi->offset); skb_frag_size_set(frag, frag_consumed_bytes); - if (page_is_pfmemalloc(di->page)) + if (page_is_pfmemalloc(au->page)) xdp_buff_set_frag_pfmemalloc(&xdp); sinfo->xdp_frags_size += frag_consumed_bytes; @@ -1661,10 +1664,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi wi++; } - di = head_wi->di; + au = head_wi->au; prog = rcu_dereference(rq->xdp_prog); - if (prog && mlx5e_xdp_handle(rq, di->page, prog, &xdp)) { + if (prog && mlx5e_xdp_handle(rq, au->page, prog, &xdp)) { if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { int i; @@ -1681,7 +1684,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (unlikely(!skb)) return NULL; - page_ref_inc(di->page); + page_ref_inc(au->page); if (unlikely(xdp_buff_has_frags(&xdp))) { int i; @@ -1876,8 +1879,8 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = { #endif static void -mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_info *di, - u32 data_bcnt, u32 data_offset) +mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, + struct mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset) { net_prefetchw(skb->data); @@ -1891,12 +1894,12 @@ mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_i else truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); - mlx5e_add_skb_frag(rq, skb, di, data_offset, + mlx5e_add_skb_frag(rq, skb, au, data_offset, pg_consumed_bytes, truesize); data_bcnt -= pg_consumed_bytes; data_offset = 0; - di++; + au++; } } @@ -1904,11 +1907,11 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { + struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); - struct mlx5e_dma_info *di = &wi->dma_info[page_idx]; u32 frag_offset = head_offset + headlen; u32 byte_cnt = cqe_bcnt - headlen; - struct mlx5e_dma_info *head_di = di; + struct mlx5e_alloc_unit *head_au = au; struct sk_buff *skb; skb = napi_alloc_skb(rq->cq.napi, @@ -1922,13 +1925,14 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ if (unlikely(frag_offset >= PAGE_SIZE)) { - di++; + au++; frag_offset -= PAGE_SIZE; } - mlx5e_fill_skb_data(skb, rq, di, byte_cnt, frag_offset); + mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset); /* copy header */ - mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, head_offset, headlen); + mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, head_au->addr, + head_offset, head_offset, headlen); /* skb linear part was allocated with headlen and aligned to long */ skb->tail += headlen; skb->len += headlen; @@ -1940,7 +1944,7 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { - struct mlx5e_dma_info *di = &wi->dma_info[page_idx]; + struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; @@ -1954,11 +1958,11 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return NULL; } - va = page_address(di->page) + head_offset; + va = page_address(au->page) + head_offset; data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset, + dma_sync_single_range_for_cpu(rq->pdev, au->addr, head_offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); @@ -1968,7 +1972,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp); - if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) { + if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */ return NULL; /* page/packet was consumed by XDP */ @@ -1984,7 +1988,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return NULL; /* queue up for recycling/reuse */ - page_ref_inc(di->page); + page_ref_inc(au->page); return skb; } @@ -2029,7 +2033,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, } prefetchw(skb->data); - mlx5e_copy_skb_header(rq->pdev, skb, head, + mlx5e_copy_skb_header(rq->pdev, skb, head->page, head->addr, head_offset + rx_headroom, rx_headroom, head_size); /* skb linear part was allocated with headlen and aligned to long */ @@ -2080,8 +2084,12 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) u64 addr = shampo->info[header_index].addr; if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) { + struct mlx5e_alloc_unit au = { + .page = shampo->info[header_index].page, + }; + shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE); - mlx5e_page_release(rq, &shampo->info[header_index], true); + mlx5e_page_release(rq, &au, true); } bitmap_clear(shampo->bitmap, header_index, 1); } @@ -2102,7 +2110,7 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq bool match = cqe->shampo.match; struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; - struct mlx5e_dma_info *di; + struct mlx5e_alloc_unit *au; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; @@ -2152,8 +2160,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq } if (likely(head_size)) { - di = &wi->dma_info[page_idx]; - mlx5e_fill_skb_data(*skb, rq, di, data_bcnt, data_offset); + au = &wi->alloc_units[page_idx]; + mlx5e_fill_skb_data(*skb, rq, au, data_bcnt, data_offset); } mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb); From 6bdeb963822af244dc4b4341d4e65a9ec6fda668 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:51 -0700 Subject: [PATCH 11/16] net/mlx5e: Remove DMA address from mlx5e_alloc_unit mlx5e_alloc_unit stores the DMA address and a pointer to either struct page (regular RQ) or struct xdp_buff (XSK RQ). This DMA address is redundant, because when a page or an XSK frame is allocated, the same address is also stored there. Some flows take the address from struct mlx5e_alloc_unit, and some take it from struct page or xdp_buff. This commit removes the address from struct mlx5e_alloc_unit, which makes it twice as small and improves locality (this struct is used in an array), also saving on unnecessary stores to the addr field. Almost all flows know unambiguously whether the DMA address should be taken from page or from xdp_buff. The exception is the allocation flows, where a new branch appeared, which will be optimized out in the next commits. struct mlx5e_alloc_unit used to be called mlx5e_dma_info. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 1 - .../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 7 --- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 57 +++++++++++++------ 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 06f49ef71b5b..944bf4c92582 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -475,7 +475,6 @@ struct mlx5e_txqsq { } ____cacheline_aligned_in_smp; struct mlx5e_alloc_unit { - dma_addr_t addr; union { struct page *page; struct xdp_buff *xsk; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index c9bdd2c04922..d6dddad890e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -25,13 +25,6 @@ static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, if (!au->xsk) return -ENOMEM; - /* Store the DMA address without headroom. In striding RQ case, we just - * provide pages for UMR, and headroom is counted at the setup stage - * when creating a WQE. In non-striding RQ case, headroom is accounted - * in mlx5e_alloc_rx_wqe. - */ - au->addr = xsk_buff_xdp_get_frame_dma(au->xsk); - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 83941a4c40a1..a3891a88863e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -254,6 +254,7 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_un { struct mlx5e_page_cache *cache = &rq->page_cache; struct mlx5e_rq_stats *stats = rq->stats; + dma_addr_t addr; if (unlikely(cache->head == cache->tail)) { stats->cache_empty++; @@ -266,17 +267,19 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_un } au->page = cache->page_cache[cache->head]; - au->addr = page_pool_get_dma_addr(au->page); cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1); stats->cache_reuse++; + addr = page_pool_get_dma_addr(au->page); /* Non-XSK always uses PAGE_SIZE. */ - dma_sync_single_for_device(rq->pdev, au->addr, PAGE_SIZE, DMA_FROM_DEVICE); + dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, DMA_FROM_DEVICE); return true; } static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) { + dma_addr_t addr; + if (mlx5e_rx_cache_get(rq, au)) return 0; @@ -285,14 +288,14 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_ return -ENOMEM; /* Non-XSK always uses PAGE_SIZE. */ - au->addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE, - rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); - if (unlikely(dma_mapping_error(rq->pdev, au->addr))) { + addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE, + rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); + if (unlikely(dma_mapping_error(rq->pdev, addr))) { page_pool_recycle_direct(rq->page_pool, au->page); au->page = NULL; return -ENOMEM; } - page_pool_set_dma_addr(au->page, au->addr); + page_pool_set_dma_addr(au->page, addr); return 0; } @@ -380,6 +383,7 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, int i; for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) { + dma_addr_t addr; u16 headroom; err = mlx5e_get_rx_frag(rq, frag); @@ -387,8 +391,9 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, goto free_frags; headroom = i == 0 ? rq->buff.headroom : 0; - wqe->data[i].addr = cpu_to_be64(frag->au->addr + - frag->offset + headroom); + addr = rq->xsk_pool ? xsk_buff_xdp_get_frame_dma(frag->au->xsk) : + page_pool_get_dma_addr(frag->au->page); + wqe->data[i].addr = cpu_to_be64(addr + frag->offset + headroom); } return 0; @@ -456,8 +461,9 @@ mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, struct mlx5e_alloc_unit *au, u32 frag_offset, u32 len, unsigned int truesize) { - dma_sync_single_for_cpu(rq->pdev, au->addr + frag_offset, - len, DMA_FROM_DEVICE); + dma_addr_t addr = page_pool_get_dma_addr(au->page); + + dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, DMA_FROM_DEVICE); page_ref_inc(au->page); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, au->page, frag_offset, len, truesize); @@ -698,21 +704,29 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) if (unlikely(rq->mpwqe.unaligned)) { for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) { + dma_addr_t addr; + err = mlx5e_page_alloc(rq, au); if (unlikely(err)) goto err_unmap; + /* Unaligned means XSK. */ + addr = xsk_buff_xdp_get_frame_dma(au->xsk); umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { .key = rq->mkey_be, - .va = cpu_to_be64(au->addr), + .va = cpu_to_be64(addr), }; } } else { for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) { + dma_addr_t addr; + err = mlx5e_page_alloc(rq, au); if (unlikely(err)) goto err_unmap; + addr = rq->xsk_pool ? xsk_buff_xdp_get_frame_dma(au->xsk) : + page_pool_get_dma_addr(au->page); umr_wqe->inline_mtts[i] = (struct mlx5_mtt) { - .ptag = cpu_to_be64(au->addr | MLX5_EN_WR), + .ptag = cpu_to_be64(addr | MLX5_EN_WR), }; } } @@ -1563,13 +1577,15 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, struct sk_buff *skb; u32 metasize = 0; void *va, *data; + dma_addr_t addr; u32 frag_size; va = page_address(au->page) + wi->offset; data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); @@ -1610,13 +1626,15 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi struct bpf_prog *prog; struct xdp_buff xdp; struct sk_buff *skb; + dma_addr_t addr; u32 truesize; void *va; va = page_address(au->page) + wi->offset; frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, rq->buff.frame0_sz, DMA_FROM_DEVICE); net_prefetchw(va); /* xdp_frame data area */ net_prefetch(va + rx_headroom); @@ -1636,7 +1654,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_for_cpu(rq->pdev, au->addr + wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_for_cpu(rq->pdev, addr + wi->offset, frag_consumed_bytes, DMA_FROM_DEVICE); if (!xdp_buff_has_frags(&xdp)) { @@ -1913,6 +1932,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w u32 byte_cnt = cqe_bcnt - headlen; struct mlx5e_alloc_unit *head_au = au; struct sk_buff *skb; + dma_addr_t addr; skb = napi_alloc_skb(rq->cq.napi, ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long))); @@ -1931,7 +1951,8 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset); /* copy header */ - mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, head_au->addr, + addr = page_pool_get_dma_addr(head_au->page); + mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, addr, head_offset, head_offset, headlen); /* skb linear part was allocated with headlen and aligned to long */ skb->tail += headlen; @@ -1950,6 +1971,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, struct sk_buff *skb; u32 metasize = 0; void *va, *data; + dma_addr_t addr; u32 frag_size; /* Check packet size. Note LRO doesn't use linear SKB */ @@ -1962,7 +1984,8 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, au->addr, head_offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, head_offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); From 672db0243349d3510e2c7f9942e97945a296f73e Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:52 -0700 Subject: [PATCH 12/16] net/mlx5e: Convert struct mlx5e_alloc_unit to a union struct mlx5e_alloc_unit consists of a single union. Convert it to a union itself to simplify casting it to struct xdp_buff *, which will be used to implement XSK batching on striding RQ. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 14 ++++---- .../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 2 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 10 +++--- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 36 +++++++++---------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 944bf4c92582..95a232fb2127 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -474,11 +474,9 @@ struct mlx5e_txqsq { cqe_ts_to_ns ptp_cyc2time; } ____cacheline_aligned_in_smp; -struct mlx5e_alloc_unit { - union { - struct page *page; - struct xdp_buff *xsk; - }; +union mlx5e_alloc_unit { + struct page *page; + struct xdp_buff *xsk; }; /* XDP packets can be transmitted in different ways. On completion, we need to @@ -607,7 +605,7 @@ struct mlx5e_icosq { } ____cacheline_aligned_in_smp; struct mlx5e_wqe_frag_info { - struct mlx5e_alloc_unit *au; + union mlx5e_alloc_unit *au; u32 offset; bool last_in_page; }; @@ -615,7 +613,7 @@ struct mlx5e_wqe_frag_info { struct mlx5e_mpw_info { u16 consumed_strides; DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE); - struct mlx5e_alloc_unit alloc_units[]; + union mlx5e_alloc_unit alloc_units[]; }; #define MLX5E_MAX_RX_FRAGS 4 @@ -694,7 +692,7 @@ struct mlx5e_rq { struct { struct mlx5_wq_cyc wq; struct mlx5e_wqe_frag_info *frags; - struct mlx5e_alloc_unit *alloc_units; + union mlx5e_alloc_unit *alloc_units; struct mlx5e_rq_frags_info info; mlx5e_fp_skb_from_cqe skb_from_cqe; } wqe; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index d6dddad890e9..53a833c9b09e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -19,7 +19,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, u32 cqe_bcnt); static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, - struct mlx5e_alloc_unit *au) + union mlx5e_alloc_unit *au) { au->xsk = xsk_buff_alloc(rq->xsk_pool); if (!au->xsk) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 47d60a53727f..b9591f902760 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -460,7 +460,7 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) prev->last_in_page = true; } -static int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) +static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node) { int len = wq_sz << rq->wqe.info.log_num_frags; @@ -474,7 +474,7 @@ static int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) return 0; } -static void mlx5e_free_di_list(struct mlx5e_rq *rq) +static void mlx5e_free_au_list(struct mlx5e_rq *rq) { kvfree(rq->wqe.alloc_units); } @@ -693,7 +693,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, goto err_rq_wq_destroy; } - err = mlx5e_init_di_list(rq, wq_sz, node); + err = mlx5e_init_au_list(rq, wq_sz, node); if (err) goto err_rq_frags; } @@ -792,7 +792,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, mlx5e_free_mpwqe_rq_drop_page(rq); break; default: /* MLX5_WQ_TYPE_CYCLIC */ - mlx5e_free_di_list(rq); + mlx5e_free_au_list(rq); err_rq_frags: kvfree(rq->wqe.frags); } @@ -826,7 +826,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) break; default: /* MLX5_WQ_TYPE_CYCLIC */ kvfree(rq->wqe.frags); - mlx5e_free_di_list(rq); + mlx5e_free_au_list(rq); } for (i = rq->page_cache.head; i != rq->page_cache.tail; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index a3891a88863e..0d0064d66c09 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -250,7 +250,7 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) return true; } -static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) +static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { struct mlx5e_page_cache *cache = &rq->page_cache; struct mlx5e_rq_stats *stats = rq->stats; @@ -276,7 +276,7 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_un return true; } -static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) +static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { dma_addr_t addr; @@ -300,7 +300,7 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_ return 0; } -static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au) +static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { if (rq->xsk_pool) return mlx5e_xsk_page_alloc_pool(rq, au); @@ -333,7 +333,7 @@ void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool rec } static inline void mlx5e_page_release(struct mlx5e_rq *rq, - struct mlx5e_alloc_unit *au, + union mlx5e_alloc_unit *au, bool recycle) { if (rq->xsk_pool) @@ -458,7 +458,7 @@ static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk) static inline void mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, - struct mlx5e_alloc_unit *au, u32 frag_offset, u32 len, + union mlx5e_alloc_unit *au, u32 frag_offset, u32 len, unsigned int truesize) { dma_addr_t addr = page_pool_get_dma_addr(au->page); @@ -485,7 +485,7 @@ mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb, static void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle) { - struct mlx5e_alloc_unit *alloc_units = wi->alloc_units; + union mlx5e_alloc_unit *alloc_units = wi->alloc_units; bool no_xdp_xmit; int i; @@ -582,7 +582,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) << MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; if (!(header_offset & (PAGE_SIZE - 1))) { - struct mlx5e_alloc_unit au; + union mlx5e_alloc_unit au; err = mlx5e_page_alloc(rq, &au); if (unlikely(err)) @@ -620,7 +620,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, while (--i >= 0) { dma_info = &shampo->info[--index]; if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) { - struct mlx5e_alloc_unit au = { + union mlx5e_alloc_unit au = { .page = dma_info->page, }; @@ -674,7 +674,7 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); - struct mlx5e_alloc_unit *au = &wi->alloc_units[0]; + union mlx5e_alloc_unit *au = &wi->alloc_units[0]; struct mlx5e_icosq *sq = rq->icosq; struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5e_umr_wqe *umr_wqe; @@ -791,7 +791,7 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close hd_info = &shampo->info[index]; hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE); if (hd_info->page != deleted_page) { - struct mlx5e_alloc_unit au = { + union mlx5e_alloc_unit au = { .page = hd_info->page, }; @@ -1571,7 +1571,7 @@ static struct sk_buff * mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { - struct mlx5e_alloc_unit *au = wi->au; + union mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; @@ -1619,7 +1619,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; struct mlx5e_wqe_frag_info *head_wi = wi; - struct mlx5e_alloc_unit *au = wi->au; + union mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; struct skb_shared_info *sinfo; u32 frag_consumed_bytes; @@ -1899,7 +1899,7 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = { static void mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, - struct mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset) + union mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset) { net_prefetchw(skb->data); @@ -1926,11 +1926,11 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { - struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; + union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); u32 frag_offset = head_offset + headlen; u32 byte_cnt = cqe_bcnt - headlen; - struct mlx5e_alloc_unit *head_au = au; + union mlx5e_alloc_unit *head_au = au; struct sk_buff *skb; dma_addr_t addr; @@ -1965,7 +1965,7 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { - struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; + union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; @@ -2107,7 +2107,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) u64 addr = shampo->info[header_index].addr; if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) { - struct mlx5e_alloc_unit au = { + union mlx5e_alloc_unit au = { .page = shampo->info[header_index].page, }; @@ -2133,7 +2133,7 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq bool match = cqe->shampo.match; struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; - struct mlx5e_alloc_unit *au; + union mlx5e_alloc_unit *au; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; From 2d0765f78c130ce692097de8a5e30a938ee41102 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:53 -0700 Subject: [PATCH 13/16] net/mlx5e: xsk: Remove mlx5e_xsk_page_alloc_pool mlx5e_xsk_page_alloc_pool became a thin wrapper around xsk_buff_alloc. Drop it and call xsk_buff_alloc directly. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h | 10 ---------- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 8 +++++--- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index 53a833c9b09e..e702cb790476 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -18,16 +18,6 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt); -static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, - union mlx5e_alloc_unit *au) -{ - au->xsk = xsk_buff_alloc(rq->xsk_pool); - if (!au->xsk) - return -ENOMEM; - - return 0; -} - static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err) { if (!xsk_uses_need_wakeup(rq->xsk_pool)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 0d0064d66c09..72d74de3ee99 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -302,10 +302,12 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_u static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { - if (rq->xsk_pool) - return mlx5e_xsk_page_alloc_pool(rq, au); - else + if (rq->xsk_pool) { + au->xsk = xsk_buff_alloc(rq->xsk_pool); + return likely(au->xsk) ? 0 : -ENOMEM; + } else { return mlx5e_page_alloc_pool(rq, au); + } } void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page) From d32c225316d407109aa8ea9835863f124aedea28 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:54 -0700 Subject: [PATCH 14/16] net/mlx5e: Split out channel (de)activation in rx_res To decrease the nesting level and reduce duplication of code, create functions to redirect direct RQTs to the actual RQs or drop_rq, which are used in the activation and deactivation flows of channels. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- .../ethernet/mellanox/mlx5/core/en/rx_res.c | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index 24c32f73040a..3436ecfcbc2f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -523,6 +523,53 @@ static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int i return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt); } +static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, + struct mlx5e_channels *chs, + unsigned int ix) +{ + u32 rqn; + int err; + + mlx5e_channels_get_regular_rqn(chs, ix, &rqn); + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), + rqn, ix, err); + + if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) + return; + + if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn)) + rqn = res->drop_rqn; + err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), + rqn, ix, err); +} + +static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, + unsigned int ix) +{ + int err; + + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), + res->drop_rqn, ix, err); + + if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) + return; + + err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), + res->drop_rqn, ix, err); +} + void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs) { unsigned int nch, ix; @@ -536,43 +583,10 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann mlx5e_rx_res_rss_enable(res); - for (ix = 0; ix < nch; ix++) { - u32 rqn; - - mlx5e_channels_get_regular_rqn(chs, ix, &rqn); - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - rqn, ix, err); - - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; - - if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn)) - rqn = res->drop_rqn; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - rqn, ix, err); - } - for (ix = nch; ix < res->max_nch; ix++) { - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - res->drop_rqn, ix, err); - - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; - - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - res->drop_rqn, ix, err); - } + for (ix = 0; ix < nch; ix++) + mlx5e_rx_res_channel_activate_direct(res, chs, ix); + for (ix = nch; ix < res->max_nch; ix++) + mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { u32 rqn; @@ -595,22 +609,8 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) mlx5e_rx_res_rss_disable(res); - for (ix = 0; ix < res->max_nch; ix++) { - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - res->drop_rqn, ix, err); - - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; - - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - res->drop_rqn, ix, err); - } + for (ix = 0; ix < res->max_nch; ix++) + mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); From e64d71d055ca01fa5054d25b99fb29b98e543a31 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:55 -0700 Subject: [PATCH 15/16] net/mlx5e: Move repeating clear_bit in mlx5e_rx_reporter_err_rq_cqe_recover The same clear_bit is called in both error and success flows. Move the call to do it only once and remove the out label. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index fc366e66d0b0..2b946ae1d97f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -162,10 +162,10 @@ static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) mlx5e_free_rx_descs(rq); err = mlx5e_rq_to_ready(rq, MLX5_RQC_STATE_ERR); - if (err) - goto out; - clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); + if (err) + return err; + mlx5e_activate_rq(rq); rq->stats->recover++; if (rq->channel) @@ -173,9 +173,6 @@ static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(rq->cq.napi); return 0; -out: - clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); - return err; } static int mlx5e_rx_reporter_timeout_recover(void *ctx) From 8f5ed1c140f8437be2881e4388d7ecfdb9a9b5ec Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Thu, 29 Sep 2022 00:21:56 -0700 Subject: [PATCH 16/16] net/mlx5e: Clean up and fix error flows in mlx5e_alloc_rq Although mlx5e_rq_free_shampo can be called unconditionally, it belongs to case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ. Move it there to allow to add more init/cleanup actions to the striding RQ case. If xdp_rxq_info_reg_mem_model fails, don't forget to destroy the page pool. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Saeed Mahameed Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b9591f902760..2719247b18db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -668,7 +668,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); if (err) - goto err_free_by_rq_type; + goto err_free_mpwqe_info; break; default: /* MLX5_WQ_TYPE_CYCLIC */ @@ -720,14 +720,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, if (IS_ERR(rq->page_pool)) { err = PTR_ERR(rq->page_pool); rq->page_pool = NULL; - goto err_free_shampo; + goto err_free_by_rq_type; } if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, rq->page_pool); } if (err) - goto err_free_shampo; + goto err_destroy_page_pool; for (i = 0; i < wq_sz; i++) { if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { @@ -780,11 +780,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, return 0; -err_free_shampo: - mlx5e_rq_free_shampo(rq); +err_destroy_page_pool: + page_pool_destroy(rq->page_pool); err_free_by_rq_type: switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + mlx5e_rq_free_shampo(rq); +err_free_mpwqe_info: kvfree(rq->mpwqe.info); err_rq_mkey: mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));