From 65131ea8d3f9a6033753d853ab497b61a2fd6090 Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:39 +0100 Subject: [PATCH 1/7] sfc: remove obsolete counters from struct efx_channel The n_rx_tobe_disc and n_rx_mcast_mismatch counters are a legacy from farch, and are never written in EF10 or EF100 code. Remove them from the struct and from ethtool -S output, saving a bit of memory and avoiding user confusion. Reviewed-by: Jacob Keller Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/ethtool_common.c | 2 -- drivers/net/ethernet/sfc/net_driver.h | 4 ---- 2 files changed, 6 deletions(-) diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index 6ded44b86052..a8baeacd83c0 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -75,7 +75,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { EFX_ETHTOOL_UINT_TXQ_STAT(pio_packets), EFX_ETHTOOL_UINT_TXQ_STAT(cb_packets), EFX_ETHTOOL_ATOMIC_NIC_ERROR_STAT(rx_reset), - EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_tobe_disc), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_ip_hdr_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_tcp_udp_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_inner_ip_hdr_chksum_err), @@ -83,7 +82,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_outer_ip_hdr_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_outer_tcp_udp_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_eth_crc_err), - EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_mcast_mismatch), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_frm_trunc), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_events), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_packets), diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index b85c51cbe7f9..4d904e1404d4 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -451,10 +451,8 @@ enum efx_sync_events_state { * @filter_work: Work item for efx_filter_rfs_expire() * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS, * indexed by filter ID - * @n_rx_tobe_disc: Count of RX_TOBE_DISC errors * @n_rx_ip_hdr_chksum_err: Count of RX IP header checksum errors * @n_rx_tcp_udp_chksum_err: Count of RX TCP and UDP checksum errors - * @n_rx_mcast_mismatch: Count of unmatched multicast frames * @n_rx_frm_trunc: Count of RX_FRM_TRUNC errors * @n_rx_overlength: Count of RX_OVERLENGTH errors * @n_skbuff_leaks: Count of skbuffs leaked due to RX overrun @@ -511,7 +509,6 @@ struct efx_channel { u32 *rps_flow_id; #endif - unsigned int n_rx_tobe_disc; unsigned int n_rx_ip_hdr_chksum_err; unsigned int n_rx_tcp_udp_chksum_err; unsigned int n_rx_outer_ip_hdr_chksum_err; @@ -519,7 +516,6 @@ struct efx_channel { unsigned int n_rx_inner_ip_hdr_chksum_err; unsigned int n_rx_inner_tcp_udp_chksum_err; unsigned int n_rx_eth_crc_err; - unsigned int n_rx_mcast_mismatch; unsigned int n_rx_frm_trunc; unsigned int n_rx_overlength; unsigned int n_skbuff_leaks; From 873e85795026e1c57dd6db24148fc5b85cc2ccee Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:40 +0100 Subject: [PATCH 2/7] sfc: implement basic per-queue stats Just RX and TX packet counts and TX bytes for now. We do not have per-queue RX byte counts, which causes us to fail stats.pkt_byte_sum selftest with "Drivers should always report basic keys" error. Per-queue counts are since the last time the queue was inited (typically by efx_start_datapath(), on ifup or reconfiguration); device-wide total (efx_get_base_stats()) is since driver probe. This is not the same lifetime as rtnl_link_stats64, which uses firmware stats which count since FW (re)booted; this can cause a "Qstats are lower" or "RTNL stats are lower" failure in stats.pkt_byte_sum selftest. Move the increment of rx_queue->rx_packets to match the semantics specified for netdev per-queue stats, i.e. just before handing the packet to XDP (if present) or the netstack (through GRO). This will affect the existing ethtool -S output which also reports these counters. XDP TX packets are not yet counted into base_stats. Signed-off-by: Edward Cree Reviewed-by: Jacob Keller Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/ef100_rx.c | 4 +- drivers/net/ethernet/sfc/efx.c | 75 +++++++++++++++++++++++++ drivers/net/ethernet/sfc/efx_channels.c | 2 + drivers/net/ethernet/sfc/net_driver.h | 22 ++++++++ drivers/net/ethernet/sfc/rx.c | 4 +- drivers/net/ethernet/sfc/rx_common.c | 2 + drivers/net/ethernet/sfc/tx_common.c | 3 + 7 files changed, 108 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c index 83d9db71d7d7..992151775cb8 100644 --- a/drivers/net/ethernet/sfc/ef100_rx.c +++ b/drivers/net/ethernet/sfc/ef100_rx.c @@ -134,6 +134,8 @@ void __ef100_rx_packet(struct efx_channel *channel) goto free_rx_buffer; } + ++rx_queue->rx_packets; + efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, csum); goto out; @@ -149,8 +151,6 @@ static void ef100_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index) struct efx_channel *channel = efx_rx_queue_channel(rx_queue); struct efx_nic *efx = rx_queue->efx; - ++rx_queue->rx_packets; - netif_vdbg(efx, rx_status, efx->net_dev, "RX queue %d received id %x\n", efx_rx_queue_index(rx_queue), index); diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 36b3b57e2055..21dc4b885542 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -22,6 +22,7 @@ #include "net_driver.h" #include #include +#include #include "efx.h" #include "efx_common.h" #include "efx_channels.h" @@ -626,6 +627,79 @@ static const struct net_device_ops efx_netdev_ops = { .ndo_bpf = efx_xdp }; +static void efx_get_queue_stats_rx(struct net_device *net_dev, int idx, + struct netdev_queue_stats_rx *stats) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + struct efx_rx_queue *rx_queue; + struct efx_channel *channel; + + channel = efx_get_channel(efx, idx); + rx_queue = efx_channel_get_rx_queue(channel); + /* Count only packets since last time datapath was started */ + stats->packets = rx_queue->rx_packets - rx_queue->old_rx_packets; +} + +static void efx_get_queue_stats_tx(struct net_device *net_dev, int idx, + struct netdev_queue_stats_tx *stats) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + struct efx_tx_queue *tx_queue; + struct efx_channel *channel; + + channel = efx_get_tx_channel(efx, idx); + stats->packets = 0; + stats->bytes = 0; + efx_for_each_channel_tx_queue(tx_queue, channel) { + stats->packets += tx_queue->complete_packets - + tx_queue->old_complete_packets; + stats->bytes += tx_queue->complete_bytes - + tx_queue->old_complete_bytes; + } +} + +static void efx_get_base_stats(struct net_device *net_dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + struct efx_tx_queue *tx_queue; + struct efx_rx_queue *rx_queue; + struct efx_channel *channel; + + rx->packets = 0; + tx->packets = 0; + tx->bytes = 0; + + /* Count all packets on non-core queues, and packets before last + * datapath start on core queues. + */ + efx_for_each_channel(channel, efx) { + rx_queue = efx_channel_get_rx_queue(channel); + if (channel->channel >= net_dev->real_num_rx_queues) + rx->packets += rx_queue->rx_packets; + else + rx->packets += rx_queue->old_rx_packets; + efx_for_each_channel_tx_queue(tx_queue, channel) { + if (channel->channel < efx->tx_channel_offset || + channel->channel >= efx->tx_channel_offset + + net_dev->real_num_tx_queues) { + tx->packets += tx_queue->complete_packets; + tx->bytes += tx_queue->complete_bytes; + } else { + tx->packets += tx_queue->old_complete_packets; + tx->bytes += tx_queue->old_complete_bytes; + } + } + } +} + +static const struct netdev_stat_ops efx_stat_ops = { + .get_queue_stats_rx = efx_get_queue_stats_rx, + .get_queue_stats_tx = efx_get_queue_stats_tx, + .get_base_stats = efx_get_base_stats, +}; + static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog) { struct bpf_prog *old_prog; @@ -716,6 +790,7 @@ static int efx_register_netdev(struct efx_nic *efx) net_dev->watchdog_timeo = 5 * HZ; net_dev->irq = efx->pci_dev->irq; net_dev->netdev_ops = &efx_netdev_ops; + net_dev->stat_ops = &efx_stat_ops; if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) net_dev->priv_flags |= IFF_UNICAST_FLT; net_dev->ethtool_ops = &efx_ethtool_ops; diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index c9e17a8208a9..834d51812e2b 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -1209,6 +1209,8 @@ static int efx_process_channel(struct efx_channel *channel, int budget) tx_queue->pkts_compl, tx_queue->bytes_compl); } + tx_queue->complete_packets += tx_queue->pkts_compl; + tx_queue->complete_bytes += tx_queue->bytes_compl; } /* Receive any packets we queued up */ diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 4d904e1404d4..83c33c1ca120 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -193,6 +193,10 @@ struct efx_tx_buffer { * @initialised: Has hardware queue been initialised? * @timestamping: Is timestamping enabled for this channel? * @xdp_tx: Is this an XDP tx queue? + * @old_complete_packets: Value of @complete_packets as of last + * efx_init_tx_queue() + * @old_complete_bytes: Value of @complete_bytes as of last + * efx_init_tx_queue() * @read_count: Current read pointer. * This is the number of buffers that have been removed from both rings. * @old_write_count: The value of @write_count when last checked. @@ -202,6 +206,16 @@ struct efx_tx_buffer { * avoid cache-line ping-pong between the xmit path and the * completion path. * @merge_events: Number of TX merged completion events + * @bytes_compl: Number of bytes completed during this NAPI poll + * (efx_process_channel()). For BQL. + * @pkts_compl: Number of packets completed during this NAPI poll. + * @complete_packets: Number of packets completed since this struct was + * created. Only counts SKB packets, not XDP TX (it accumulates + * the same values that are reported to BQL). + * @complete_bytes: Number of bytes completed since this struct was + * created. For TSO, counts the superframe size, not the sizes of + * generated frames on the wire (i.e. the headers are only counted + * once) * @completed_timestamp_major: Top part of the most recent tx timestamp. * @completed_timestamp_minor: Low part of the most recent tx timestamp. * @insert_count: Current insert pointer @@ -232,6 +246,7 @@ struct efx_tx_buffer { * @xmit_pending: Are any packets waiting to be pushed to the NIC * @cb_packets: Number of times the TX copybreak feature has been used * @notify_count: Count of notified descriptors to the NIC + * @tx_packets: Number of packets sent since this struct was created * @empty_read_count: If the completion path has seen the queue as empty * and the transmission path has not yet checked this, the value of * @read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0. @@ -255,6 +270,8 @@ struct efx_tx_queue { bool initialised; bool timestamping; bool xdp_tx; + unsigned long old_complete_packets; + unsigned long old_complete_bytes; /* Members used mainly on the completion path */ unsigned int read_count ____cacheline_aligned_in_smp; @@ -262,6 +279,8 @@ struct efx_tx_queue { unsigned int merge_events; unsigned int bytes_compl; unsigned int pkts_compl; + unsigned long complete_packets; + unsigned long complete_bytes; u32 completed_timestamp_major; u32 completed_timestamp_minor; @@ -370,6 +389,8 @@ struct efx_rx_page_state { * @recycle_count: RX buffer recycle counter. * @slow_fill: Timer used to defer efx_nic_generate_fill_event(). * @grant_work: workitem used to grant credits to the MAE if @grant_credits + * @rx_packets: Number of packets received since this struct was created + * @old_rx_packets: Value of @rx_packets as of last efx_init_rx_queue() * @xdp_rxq_info: XDP specific RX queue information. * @xdp_rxq_info_valid: Is xdp_rxq_info valid data?. */ @@ -406,6 +427,7 @@ struct efx_rx_queue { struct work_struct grant_work; /* Statistics to supplement MAC stats */ unsigned long rx_packets; + unsigned long old_rx_packets; struct xdp_rxq_info xdp_rxq_info; bool xdp_rxq_info_valid; }; diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index f77a2d3ef37e..f07495582125 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -125,8 +125,6 @@ void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, struct efx_channel *channel = efx_rx_queue_channel(rx_queue); struct efx_rx_buffer *rx_buf; - rx_queue->rx_packets++; - rx_buf = efx_rx_buffer(rx_queue, index); rx_buf->flags |= flags; @@ -394,6 +392,8 @@ void __efx_rx_packet(struct efx_channel *channel) goto out; } + rx_queue->rx_packets++; + if (!efx_do_xdp(efx, channel, rx_buf, &eh)) goto out; diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index 0b7dc75c40f9..bdb4325a7c2c 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -241,6 +241,8 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) rx_queue->page_recycle_failed = 0; rx_queue->page_recycle_full = 0; + rx_queue->old_rx_packets = rx_queue->rx_packets; + /* Initialise limit fields */ max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM; max_trigger = diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c index 2adb132b2f7e..6d47927e1c2c 100644 --- a/drivers/net/ethernet/sfc/tx_common.c +++ b/drivers/net/ethernet/sfc/tx_common.c @@ -86,6 +86,9 @@ void efx_init_tx_queue(struct efx_tx_queue *tx_queue) tx_queue->completed_timestamp_major = 0; tx_queue->completed_timestamp_minor = 0; + tx_queue->old_complete_packets = tx_queue->complete_packets; + tx_queue->old_complete_bytes = tx_queue->complete_bytes; + tx_queue->xdp_tx = efx_channel_is_xdp_tx(tx_queue->channel); tx_queue->tso_version = 0; From 5c24de42f1c1d77cf876ae4b1830e1bbf89f456f Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:41 +0100 Subject: [PATCH 3/7] sfc: add n_rx_overlength to ethtool stats The previous patch changed when we increment the RX queue's rx_packets counter, to match the semantics of netdev per-queue stats. The differences between the old and new counts are scatter errors (which produce a WARN_ON) and this counter, which is incremented by efx_rx_packet__check_len() when an RX packet (which was placed in a single buffer by SG, i.e. n_frags == 1) has a length (from the RX event) which is too long to fit in the RX buffer. If this occurs, we drop the packet and fire a ratelimited netif_err(). The counter previously was not reported anywhere; add it to ethtool -S output to ensure users still have this information. Reviewed-by: Jacob Keller Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/ethtool_common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index a8baeacd83c0..ae32e08540fa 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -83,6 +83,7 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_outer_tcp_udp_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_eth_crc_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_frm_trunc), + EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_overlength), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_events), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_packets), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_xdp_drops), From cfa63b9080bce70a40d40ff0a0669cf81a2f66d1 Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:42 +0100 Subject: [PATCH 4/7] sfc: account XDP TXes in netdev base stats When we handle a TX completion for an XDP packet, it is not counted in the per-TXQ netdev stats. Record it in new internal counters, and include those in the device-wide total in efx_get_base_stats(). Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/efx.c | 3 +++ drivers/net/ethernet/sfc/net_driver.h | 6 ++++++ drivers/net/ethernet/sfc/tx.c | 6 +++++- drivers/net/ethernet/sfc/tx_common.c | 28 +++++++++++++++++++++------ drivers/net/ethernet/sfc/tx_common.h | 4 +++- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 21dc4b885542..ea1e0e8ecbdd 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -690,6 +690,9 @@ static void efx_get_base_stats(struct net_device *net_dev, tx->packets += tx_queue->old_complete_packets; tx->bytes += tx_queue->old_complete_bytes; } + /* Include XDP TX in device-wide stats */ + tx->packets += tx_queue->complete_xdp_packets; + tx->bytes += tx_queue->complete_xdp_bytes; } } } diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 83c33c1ca120..aba106d03d41 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -216,6 +216,10 @@ struct efx_tx_buffer { * created. For TSO, counts the superframe size, not the sizes of * generated frames on the wire (i.e. the headers are only counted * once) + * @complete_xdp_packets: Number of XDP TX packets completed since this + * struct was created. + * @complete_xdp_bytes: Number of XDP TX bytes completed since this + * struct was created. * @completed_timestamp_major: Top part of the most recent tx timestamp. * @completed_timestamp_minor: Low part of the most recent tx timestamp. * @insert_count: Current insert pointer @@ -281,6 +285,8 @@ struct efx_tx_queue { unsigned int pkts_compl; unsigned long complete_packets; unsigned long complete_bytes; + unsigned long complete_xdp_packets; + unsigned long complete_xdp_bytes; u32 completed_timestamp_major; u32 completed_timestamp_minor; diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index fe2d476028e7..822ec6564b2d 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -553,6 +553,7 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, void efx_xmit_done_single(struct efx_tx_queue *tx_queue) { + unsigned int xdp_pkts_compl = 0, xdp_bytes_compl = 0; unsigned int pkts_compl = 0, bytes_compl = 0; unsigned int efv_pkts_compl = 0; unsigned int read_ptr; @@ -577,7 +578,8 @@ void efx_xmit_done_single(struct efx_tx_queue *tx_queue) if (buffer->flags & EFX_TX_BUF_SKB) finished = true; efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, - &efv_pkts_compl); + &efv_pkts_compl, &xdp_pkts_compl, + &xdp_bytes_compl); ++tx_queue->read_count; read_ptr = tx_queue->read_count & tx_queue->ptr_mask; @@ -585,6 +587,8 @@ void efx_xmit_done_single(struct efx_tx_queue *tx_queue) tx_queue->pkts_compl += pkts_compl; tx_queue->bytes_compl += bytes_compl; + tx_queue->complete_xdp_packets += xdp_pkts_compl; + tx_queue->complete_xdp_bytes += xdp_bytes_compl; EFX_WARN_ON_PARANOID(pkts_compl + efv_pkts_compl != 1); diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c index 6d47927e1c2c..2013a609f9be 100644 --- a/drivers/net/ethernet/sfc/tx_common.c +++ b/drivers/net/ethernet/sfc/tx_common.c @@ -112,12 +112,14 @@ void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) /* Free any buffers left in the ring */ while (tx_queue->read_count != tx_queue->write_count) { + unsigned int xdp_pkts_compl = 0, xdp_bytes_compl = 0; unsigned int pkts_compl = 0, bytes_compl = 0; unsigned int efv_pkts_compl = 0; buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, - &efv_pkts_compl); + &efv_pkts_compl, &xdp_pkts_compl, + &xdp_bytes_compl); ++tx_queue->read_count; } @@ -153,7 +155,9 @@ void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, struct efx_tx_buffer *buffer, unsigned int *pkts_compl, unsigned int *bytes_compl, - unsigned int *efv_pkts_compl) + unsigned int *efv_pkts_compl, + unsigned int *xdp_pkts, + unsigned int *xdp_bytes) { if (buffer->unmap_len) { struct device *dma_dev = &tx_queue->efx->pci_dev->dev; @@ -198,6 +202,10 @@ void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, tx_queue->queue, tx_queue->read_count); } else if (buffer->flags & EFX_TX_BUF_XDP) { xdp_return_frame_rx_napi(buffer->xdpf); + if (xdp_pkts) + (*xdp_pkts)++; + if (xdp_bytes) + (*xdp_bytes) += buffer->xdpf->len; } buffer->len = 0; @@ -213,7 +221,9 @@ static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, unsigned int index, unsigned int *pkts_compl, unsigned int *bytes_compl, - unsigned int *efv_pkts_compl) + unsigned int *efv_pkts_compl, + unsigned int *xdp_pkts, + unsigned int *xdp_bytes) { struct efx_nic *efx = tx_queue->efx; unsigned int stop_index, read_ptr; @@ -233,7 +243,7 @@ static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, } efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl, - efv_pkts_compl); + efv_pkts_compl, xdp_pkts, xdp_bytes); ++tx_queue->read_count; read_ptr = tx_queue->read_count & tx_queue->ptr_mask; @@ -256,15 +266,18 @@ void efx_xmit_done_check_empty(struct efx_tx_queue *tx_queue) int efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) { unsigned int fill_level, pkts_compl = 0, bytes_compl = 0; + unsigned int xdp_pkts_compl = 0, xdp_bytes_compl = 0; unsigned int efv_pkts_compl = 0; struct efx_nic *efx = tx_queue->efx; EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask); efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl, - &efv_pkts_compl); + &efv_pkts_compl, &xdp_pkts_compl, &xdp_bytes_compl); tx_queue->pkts_compl += pkts_compl; tx_queue->bytes_compl += bytes_compl; + tx_queue->complete_xdp_packets += xdp_pkts_compl; + tx_queue->complete_xdp_bytes += xdp_bytes_compl; if (pkts_compl + efv_pkts_compl > 1) ++tx_queue->merge_events; @@ -293,6 +306,8 @@ int efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, unsigned int insert_count) { + unsigned int xdp_bytes_compl = 0; + unsigned int xdp_pkts_compl = 0; unsigned int efv_pkts_compl = 0; struct efx_tx_buffer *buffer; unsigned int bytes_compl = 0; @@ -303,7 +318,8 @@ void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, --tx_queue->insert_count; buffer = __efx_tx_queue_get_insert_buffer(tx_queue); efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, - &efv_pkts_compl); + &efv_pkts_compl, &xdp_pkts_compl, + &xdp_bytes_compl); } } diff --git a/drivers/net/ethernet/sfc/tx_common.h b/drivers/net/ethernet/sfc/tx_common.h index 1e9f42938aac..039eefafba23 100644 --- a/drivers/net/ethernet/sfc/tx_common.h +++ b/drivers/net/ethernet/sfc/tx_common.h @@ -20,7 +20,9 @@ void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, struct efx_tx_buffer *buffer, unsigned int *pkts_compl, unsigned int *bytes_compl, - unsigned int *efv_pkts_compl); + unsigned int *efv_pkts_compl, + unsigned int *xdp_pkts, + unsigned int *xdp_bytes); static inline bool efx_tx_buffer_in_use(struct efx_tx_buffer *buffer) { From 07e5fa5b7f43efd18040ae47569258850cf94a09 Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:43 +0100 Subject: [PATCH 5/7] sfc: implement per-queue rx drop and overrun stats Reviewed-by: Jacob Keller Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/efx.c | 15 +++++++++++++-- drivers/net/ethernet/sfc/efx_channels.c | 4 ++++ drivers/net/ethernet/sfc/efx_channels.h | 7 +++++++ drivers/net/ethernet/sfc/net_driver.h | 7 +++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index ea1e0e8ecbdd..b0ea4ca82cd8 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -638,6 +638,10 @@ static void efx_get_queue_stats_rx(struct net_device *net_dev, int idx, rx_queue = efx_channel_get_rx_queue(channel); /* Count only packets since last time datapath was started */ stats->packets = rx_queue->rx_packets - rx_queue->old_rx_packets; + stats->hw_drops = efx_get_queue_stat_rx_hw_drops(channel) - + channel->old_n_rx_hw_drops; + stats->hw_drop_overruns = channel->n_rx_nodesc_trunc - + channel->old_n_rx_hw_drop_overruns; } static void efx_get_queue_stats_tx(struct net_device *net_dev, int idx, @@ -668,6 +672,8 @@ static void efx_get_base_stats(struct net_device *net_dev, struct efx_channel *channel; rx->packets = 0; + rx->hw_drops = 0; + rx->hw_drop_overruns = 0; tx->packets = 0; tx->bytes = 0; @@ -676,10 +682,15 @@ static void efx_get_base_stats(struct net_device *net_dev, */ efx_for_each_channel(channel, efx) { rx_queue = efx_channel_get_rx_queue(channel); - if (channel->channel >= net_dev->real_num_rx_queues) + if (channel->channel >= net_dev->real_num_rx_queues) { rx->packets += rx_queue->rx_packets; - else + rx->hw_drops += efx_get_queue_stat_rx_hw_drops(channel); + rx->hw_drop_overruns += channel->n_rx_nodesc_trunc; + } else { rx->packets += rx_queue->old_rx_packets; + rx->hw_drops += channel->old_n_rx_hw_drops; + rx->hw_drop_overruns += channel->old_n_rx_hw_drop_overruns; + } efx_for_each_channel_tx_queue(tx_queue, channel) { if (channel->channel < efx->tx_channel_offset || channel->channel >= efx->tx_channel_offset + diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 834d51812e2b..44d92c0e1b63 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -1100,6 +1100,10 @@ void efx_start_channels(struct efx_nic *efx) atomic_inc(&efx->active_queues); } + /* reset per-queue stats */ + channel->old_n_rx_hw_drops = efx_get_queue_stat_rx_hw_drops(channel); + channel->old_n_rx_hw_drop_overruns = channel->n_rx_nodesc_trunc; + efx_for_each_channel_rx_queue(rx_queue, channel) { efx_init_rx_queue(rx_queue); atomic_inc(&efx->active_queues); diff --git a/drivers/net/ethernet/sfc/efx_channels.h b/drivers/net/ethernet/sfc/efx_channels.h index 46b702648721..547cf94014a3 100644 --- a/drivers/net/ethernet/sfc/efx_channels.h +++ b/drivers/net/ethernet/sfc/efx_channels.h @@ -43,6 +43,13 @@ struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel); void efx_start_channels(struct efx_nic *efx); void efx_stop_channels(struct efx_nic *efx); +static inline u64 efx_get_queue_stat_rx_hw_drops(struct efx_channel *channel) +{ + return channel->n_rx_eth_crc_err + channel->n_rx_frm_trunc + + channel->n_rx_overlength + channel->n_rx_nodesc_trunc + + channel->n_rx_mport_bad; +} + void efx_init_napi_channel(struct efx_channel *channel); void efx_init_napi(struct efx_nic *efx); void efx_fini_napi_channel(struct efx_channel *channel); diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index aba106d03d41..f6632f8185b2 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -494,6 +494,10 @@ enum efx_sync_events_state { * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP * @n_rx_mport_bad: Count of RX packets dropped because their ingress mport was * not recognised + * @old_n_rx_hw_drops: Count of all RX packets dropped for any reason as of last + * efx_start_channels() + * @old_n_rx_hw_drop_overruns: Value of @n_rx_nodesc_trunc as of last + * efx_start_channels() * @rx_pkt_n_frags: Number of fragments in next packet to be delivered by * __efx_rx_packet(), or zero if there is none * @rx_pkt_index: Ring index of first buffer for next packet to be delivered @@ -556,6 +560,9 @@ struct efx_channel { unsigned int n_rx_xdp_redirect; unsigned int n_rx_mport_bad; + unsigned int old_n_rx_hw_drops; + unsigned int old_n_rx_hw_drop_overruns; + unsigned int rx_pkt_n_frags; unsigned int rx_pkt_index; From db3067c8aab6836004a985a4236e9825b34c4dd7 Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:44 +0100 Subject: [PATCH 6/7] sfc: implement per-queue TSO (hw_gso) stats Use our existing TSO stats, which count enqueued TSO TXes. Users may expect them to count completions, as tx-packets and tx-bytes do; however, these are the counters we have, and the qstats documentation doesn't actually specify. Reviewed-by: Jacob Keller Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/efx.c | 16 ++++++++++++++++ drivers/net/ethernet/sfc/net_driver.h | 4 ++++ drivers/net/ethernet/sfc/tx_common.c | 2 ++ 3 files changed, 22 insertions(+) diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index b0ea4ca82cd8..68ddb28d3141 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -654,11 +654,21 @@ static void efx_get_queue_stats_tx(struct net_device *net_dev, int idx, channel = efx_get_tx_channel(efx, idx); stats->packets = 0; stats->bytes = 0; + stats->hw_gso_packets = 0; + stats->hw_gso_wire_packets = 0; efx_for_each_channel_tx_queue(tx_queue, channel) { stats->packets += tx_queue->complete_packets - tx_queue->old_complete_packets; stats->bytes += tx_queue->complete_bytes - tx_queue->old_complete_bytes; + /* Note that, unlike stats->packets and stats->bytes, + * these count TXes enqueued, rather than completed, + * which may not be what users expect. + */ + stats->hw_gso_packets += tx_queue->tso_bursts - + tx_queue->old_tso_bursts; + stats->hw_gso_wire_packets += tx_queue->tso_packets - + tx_queue->old_tso_packets; } } @@ -676,6 +686,8 @@ static void efx_get_base_stats(struct net_device *net_dev, rx->hw_drop_overruns = 0; tx->packets = 0; tx->bytes = 0; + tx->hw_gso_packets = 0; + tx->hw_gso_wire_packets = 0; /* Count all packets on non-core queues, and packets before last * datapath start on core queues. @@ -697,9 +709,13 @@ static void efx_get_base_stats(struct net_device *net_dev, net_dev->real_num_tx_queues) { tx->packets += tx_queue->complete_packets; tx->bytes += tx_queue->complete_bytes; + tx->hw_gso_packets += tx_queue->tso_bursts; + tx->hw_gso_wire_packets += tx_queue->tso_packets; } else { tx->packets += tx_queue->old_complete_packets; tx->bytes += tx_queue->old_complete_bytes; + tx->hw_gso_packets += tx_queue->old_tso_bursts; + tx->hw_gso_wire_packets += tx_queue->old_tso_packets; } /* Include XDP TX in device-wide stats */ tx->packets += tx_queue->complete_xdp_packets; diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index f6632f8185b2..4ca48db3e168 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -197,6 +197,8 @@ struct efx_tx_buffer { * efx_init_tx_queue() * @old_complete_bytes: Value of @complete_bytes as of last * efx_init_tx_queue() + * @old_tso_bursts: Value of @tso_bursts as of last efx_init_tx_queue() + * @old_tso_packets: Value of @tso_packets as of last efx_init_tx_queue() * @read_count: Current read pointer. * This is the number of buffers that have been removed from both rings. * @old_write_count: The value of @write_count when last checked. @@ -276,6 +278,8 @@ struct efx_tx_queue { bool xdp_tx; unsigned long old_complete_packets; unsigned long old_complete_bytes; + unsigned int old_tso_bursts; + unsigned int old_tso_packets; /* Members used mainly on the completion path */ unsigned int read_count ____cacheline_aligned_in_smp; diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c index 2013a609f9be..a22a0d634ffc 100644 --- a/drivers/net/ethernet/sfc/tx_common.c +++ b/drivers/net/ethernet/sfc/tx_common.c @@ -88,6 +88,8 @@ void efx_init_tx_queue(struct efx_tx_queue *tx_queue) tx_queue->old_complete_packets = tx_queue->complete_packets; tx_queue->old_complete_bytes = tx_queue->complete_bytes; + tx_queue->old_tso_bursts = tx_queue->tso_bursts; + tx_queue->old_tso_packets = tx_queue->tso_packets; tx_queue->xdp_tx = efx_channel_is_xdp_tx(tx_queue->channel); tx_queue->tso_version = 0; From b3411dbdaa55cffbcdfa1aaffa78f812132937fe Mon Sep 17 00:00:00 2001 From: Edward Cree Date: Mon, 30 Sep 2024 14:52:45 +0100 Subject: [PATCH 7/7] sfc: add per-queue RX bytes stats While this does add overhead to the fast path, it should be minimal as the cacheline should already be held for write from updating the queue's rx_packets stat. Reviewed-by: Jacob Keller Signed-off-by: Edward Cree Signed-off-by: David S. Miller --- drivers/net/ethernet/sfc/ef100_rx.c | 1 + drivers/net/ethernet/sfc/efx.c | 4 ++++ drivers/net/ethernet/sfc/net_driver.h | 4 ++++ drivers/net/ethernet/sfc/rx.c | 1 + drivers/net/ethernet/sfc/rx_common.c | 1 + 5 files changed, 11 insertions(+) diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c index 992151775cb8..44dc75feb162 100644 --- a/drivers/net/ethernet/sfc/ef100_rx.c +++ b/drivers/net/ethernet/sfc/ef100_rx.c @@ -135,6 +135,7 @@ void __ef100_rx_packet(struct efx_channel *channel) } ++rx_queue->rx_packets; + rx_queue->rx_bytes += rx_buf->len; efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, csum); goto out; diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 68ddb28d3141..90bb7db15519 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -638,6 +638,7 @@ static void efx_get_queue_stats_rx(struct net_device *net_dev, int idx, rx_queue = efx_channel_get_rx_queue(channel); /* Count only packets since last time datapath was started */ stats->packets = rx_queue->rx_packets - rx_queue->old_rx_packets; + stats->bytes = rx_queue->rx_bytes - rx_queue->old_rx_bytes; stats->hw_drops = efx_get_queue_stat_rx_hw_drops(channel) - channel->old_n_rx_hw_drops; stats->hw_drop_overruns = channel->n_rx_nodesc_trunc - @@ -682,6 +683,7 @@ static void efx_get_base_stats(struct net_device *net_dev, struct efx_channel *channel; rx->packets = 0; + rx->bytes = 0; rx->hw_drops = 0; rx->hw_drop_overruns = 0; tx->packets = 0; @@ -696,10 +698,12 @@ static void efx_get_base_stats(struct net_device *net_dev, rx_queue = efx_channel_get_rx_queue(channel); if (channel->channel >= net_dev->real_num_rx_queues) { rx->packets += rx_queue->rx_packets; + rx->bytes += rx_queue->rx_bytes; rx->hw_drops += efx_get_queue_stat_rx_hw_drops(channel); rx->hw_drop_overruns += channel->n_rx_nodesc_trunc; } else { rx->packets += rx_queue->old_rx_packets; + rx->bytes += rx_queue->old_rx_bytes; rx->hw_drops += channel->old_n_rx_hw_drops; rx->hw_drop_overruns += channel->old_n_rx_hw_drop_overruns; } diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 4ca48db3e168..b54662d32f55 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -400,7 +400,9 @@ struct efx_rx_page_state { * @slow_fill: Timer used to defer efx_nic_generate_fill_event(). * @grant_work: workitem used to grant credits to the MAE if @grant_credits * @rx_packets: Number of packets received since this struct was created + * @rx_bytes: Number of bytes received since this struct was created * @old_rx_packets: Value of @rx_packets as of last efx_init_rx_queue() + * @old_rx_bytes: Value of @rx_bytes as of last efx_init_rx_queue() * @xdp_rxq_info: XDP specific RX queue information. * @xdp_rxq_info_valid: Is xdp_rxq_info valid data?. */ @@ -437,7 +439,9 @@ struct efx_rx_queue { struct work_struct grant_work; /* Statistics to supplement MAC stats */ unsigned long rx_packets; + unsigned long rx_bytes; unsigned long old_rx_packets; + unsigned long old_rx_bytes; struct xdp_rxq_info xdp_rxq_info; bool xdp_rxq_info_valid; }; diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index f07495582125..ffca82207e47 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -393,6 +393,7 @@ void __efx_rx_packet(struct efx_channel *channel) } rx_queue->rx_packets++; + rx_queue->rx_bytes += rx_buf->len; if (!efx_do_xdp(efx, channel, rx_buf, &eh)) goto out; diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index bdb4325a7c2c..ab358fe13e1d 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -242,6 +242,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) rx_queue->page_recycle_full = 0; rx_queue->old_rx_packets = rx_queue->rx_packets; + rx_queue->old_rx_bytes = rx_queue->rx_bytes; /* Initialise limit fields */ max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;