mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 08:15:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

net: prevent torn reads in netdev_tc_txq

Browse Source 
netdev_set_tc_queue() (and related helpers/drivers such as
netdev_bind_sb_channel_queue(), netdev_reset_tc(), and
netdev_unbind_sb_channel()) perform separate 16-bit writes to
dev->tc_to_txq[tc].count and dev->tc_to_txq[tc].offset.

Furthermore, memset() in netdev_reset_tc() and
netdev_unbind_sb_channel() provides no guarantee of performing
full 32-bit word stores.

Concurrent lockless readers (e.g. skb_tx_hash(), netdev_txq_to_tc(),
ixgbe_select_queue(), taprio, mqprio, FPE drivers) can observe torn
values where offset and count belong to inconsistent configurations.

Redefine struct netdev_tc_txq to embed count and offset inside a union
with a u32 combined field, allowing atomic manipulation via
READ_ONCE() and WRITE_ONCE().

Update all lockless readers and writers across the kernel to use
READ_ONCE() and WRITE_ONCE() on the combined field.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20260812085440.3917924-2-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet
2026-08-12 08:54:38 +00:00
committed by Jakub Kicinski
parent e6a5d573d2
commit 21ef2d065a
11 changed files with 94 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/net/ethernet/intel/igc/igc_tsn.c
View File

@@ -183,13 +183,15 @@ static u32 igc_fpe_map_preempt_tc_to_queue(const struct igc_adapter *adapter,
u32 i, queue = 0;
for (i = 0; i < dev->num_tc; i++) {
struct netdev_tc_txq res;
u32 offset, count;
if (!(preemptible_tcs & BIT(i)))
continue;
offset = dev->tc_to_txq[i].offset;
count = dev->tc_to_txq[i].count;
res.combined = READ_ONCE(dev->tc_to_txq[i].combined);
offset = res.offset;
count = res.count;
queue |= GENMASK(offset + count - 1, offset);
}

7
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
View File

@@ -9273,10 +9273,11 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
if (sb_dev) {
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
struct net_device *vdev = sb_dev;
struct netdev_tc_txq res;
txq = vdev->tc_to_txq[tc].offset;
txq += reciprocal_scale(skb_get_hash(skb),
vdev->tc_to_txq[tc].count);
res.combined = READ_ONCE(vdev->tc_to_txq[tc].combined);
txq = res.offset;
txq += reciprocal_scale(skb_get_hash(skb), res.count);
return txq;
}

2
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@@ -3247,7 +3247,7 @@ static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv)
old_num_txqs = netdev->real_num_tx_queues;
old_ntc = netdev->num_tc ? : 1;
for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++)
old_tc_to_txq[i] = netdev->tc_to_txq[i];
old_tc_to_txq[i].combined = READ_ONCE(netdev->tc_to_txq[i].combined);
nch = priv->channels.params.num_channels;
ntc = priv->channels.params.mqprio.num_tc;

8
drivers/net/ethernet/sfc/falcon/tx.c
View File

@@ -439,8 +439,12 @@ int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
return 0;
for (tc = 0; tc < num_tc; tc++) {
net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels;
net_dev->tc_to_txq[tc].count = efx->n_tx_channels;
struct netdev_tc_txq res = {
.offset = tc * efx->n_tx_channels,
.count = efx->n_tx_channels,
};
WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined);
}
if (num_tc > net_dev->num_tc) {

8
drivers/net/ethernet/sfc/siena/tx.c
View File

@@ -380,8 +380,12 @@ int efx_siena_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
return 0;
for (tc = 0; tc < num_tc; tc++) {
net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels;
net_dev->tc_to_txq[tc].count = efx->n_tx_channels;
struct netdev_tc_txq res = {
.offset = tc * efx->n_tx_channels,
.count = efx->n_tx_channels,
};
WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined);
}
net_dev->num_tc = num_tc;

14
drivers/net/ethernet/stmicro/stmmac/stmmac_fpe.c
View File

@@ -217,8 +217,11 @@ int dwmac5_fpe_map_preemption_class(struct net_device *ndev,
* and is direct one-to-one mapping."
*/
for (u32 tc = 0; tc < num_tc; tc++) {
count = ndev->tc_to_txq[tc].count;
offset = ndev->tc_to_txq[tc].offset;
struct netdev_tc_txq res;
res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined);
count = res.count;
offset = res.offset;
if (pclass & BIT(tc))
preemptible_txqs |= GENMASK(offset + count - 1, offset);
@@ -275,8 +278,11 @@ int dwxgmac3_fpe_map_preemption_class(struct net_device *ndev,
* any of the scheduling algorithms."
*/
for (u32 tc = 0; tc < num_tc; tc++) {
count = ndev->tc_to_txq[tc].count;
offset = ndev->tc_to_txq[tc].offset;
struct netdev_tc_txq res;
res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined);
count = res.count;
offset = res.offset;
if (pclass & BIT(tc))
preemptible_txqs |= GENMASK(offset + count - 1, offset);

9
include/linux/netdevice.h
View File

@@ -832,8 +832,13 @@ struct xps_dev_maps {
#define TC_BITMASK 15
/* HW offloaded queuing disciplines txq count and offset maps */
struct netdev_tc_txq {
u16 count;
u16 offset;
union {
struct {
u16 count;
u16 offset;
};
u32 combined;
};
};
#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)

46
net/core/dev.c
View File

@@ -2652,11 +2652,13 @@ EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
*/
static void netif_setup_tc(struct net_device *dev, unsigned int txq)
{
struct netdev_tc_txq res;
int i;
struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
res.combined = READ_ONCE(dev->tc_to_txq[0].combined);
/* If TC0 is invalidated disable TC mapping */
if (tc->offset + tc->count > txq) {
if (res.offset + res.count > txq) {
netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
dev->num_tc = 0;
return;
@@ -2666,8 +2668,8 @@ static void netif_setup_tc(struct net_device *dev, unsigned int txq)
for (i = 1; i < TC_BITMASK + 1; i++) {
int q = netdev_get_prio_tc_map(dev, i);
tc = &dev->tc_to_txq[q];
if (tc->offset + tc->count > txq) {
res.combined = READ_ONCE(dev->tc_to_txq[q].combined);
if (res.offset + res.count > txq) {
netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
i, q);
netdev_set_prio_tc_map(dev, i, 0);
@@ -2683,7 +2685,10 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
/* walk through the TCs and see if it falls into any of them */
for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
if ((txq - tc->offset) < tc->count)
struct netdev_tc_txq res;
res.combined = READ_ONCE(tc->combined);
if ((txq - res.offset) < res.count)
return i;
}
@@ -3103,6 +3108,8 @@ static void netdev_unbind_all_sb_channels(struct net_device *dev)
void netdev_reset_tc(struct net_device *dev)
{
int i;
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(dev, 0);
#endif
@@ -3110,21 +3117,26 @@ void netdev_reset_tc(struct net_device *dev)
/* Reset TC configuration of device */
dev->num_tc = 0;
memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
for (i = 0; i < TC_MAX_QUEUE; i++)
WRITE_ONCE(dev->tc_to_txq[i].combined, 0);
memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
}
EXPORT_SYMBOL(netdev_reset_tc);
int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
{
struct netdev_tc_txq res = {
.count = count,
.offset = offset,
};
if (tc >= dev->num_tc)
return -EINVAL;
#ifdef CONFIG_XPS
netif_reset_xps_queues(dev, offset, count);
#endif
dev->tc_to_txq[tc].count = count;
dev->tc_to_txq[tc].offset = offset;
WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined);
return 0;
}
EXPORT_SYMBOL(netdev_set_tc_queue);
@@ -3148,11 +3160,13 @@ void netdev_unbind_sb_channel(struct net_device *dev,
struct net_device *sb_dev)
{
struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
int i;
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(sb_dev, 0);
#endif
memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
for (i = 0; i < TC_MAX_QUEUE; i++)
WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0);
memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
while (txq-- != &dev->_tx[0]) {
@@ -3175,8 +3189,12 @@ int netdev_bind_sb_channel_queue(struct net_device *dev,
return -EINVAL;
/* Record the mapping */
sb_dev->tc_to_txq[tc].count = count;
sb_dev->tc_to_txq[tc].offset = offset;
struct netdev_tc_txq res = {
.count = count,
.offset = offset,
};
WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined);
/* Provide a way for Tx queue to find the tc_to_txq map or
* XPS map for itself.
@@ -3542,9 +3560,11 @@ static u16 skb_tx_hash(const struct net_device *dev,
if (dev->num_tc) {
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
struct netdev_tc_txq res;
qoffset = sb_dev->tc_to_txq[tc].offset;
qcount = sb_dev->tc_to_txq[tc].count;
res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined);
qoffset = res.offset;
qcount = res.count;
if (unlikely(!qcount)) {
net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
sb_dev->name, qoffset, tc);

4
net/sched/sch_mqprio.c
View File

@@ -679,12 +679,14 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
rcu_read_lock();
if (cl >= TC_H_MIN_PRIORITY) {
struct net_device *dev = qdisc_dev(sch);
struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK];
struct netdev_tc_txq tc;
struct gnet_stats_queue qstats = {0};
struct gnet_stats_basic_sync bstats;
u32 qlen = 0;
int i;
tc.combined = READ_ONCE(dev->tc_to_txq[cl & TC_BITMASK].combined);
gnet_stats_basic_sync_init(&bstats);
for (i = tc.offset; i < tc.offset + tc.count; i++) {

7
net/sched/sch_mqprio_lib.c
View File

@@ -108,8 +108,11 @@ void mqprio_qopt_reconstruct(struct net_device *dev, struct tc_mqprio_qopt *qopt
memcpy(qopt->prio_tc_map, dev->prio_tc_map, sizeof(qopt->prio_tc_map));
for (tc = 0; tc < num_tc; tc++) {
qopt->count[tc] = dev->tc_to_txq[tc].count;
qopt->offset[tc] = dev->tc_to_txq[tc].offset;
struct netdev_tc_txq res;
res.combined = READ_ONCE(dev->tc_to_txq[tc].combined);
qopt->count[tc] = res.count;
qopt->offset[tc] = res.offset;
}
}
EXPORT_SYMBOL_GPL(mqprio_qopt_reconstruct);

26
net/sched/sch_taprio.c
View File

@@ -762,12 +762,13 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,
static void taprio_next_tc_txq(struct net_device *dev, int tc, int *txq)
{
int offset = dev->tc_to_txq[tc].offset;
int count = dev->tc_to_txq[tc].count;
struct netdev_tc_txq res;
res.combined = READ_ONCE(dev->tc_to_txq[tc].combined);
(*txq)++;
if (*txq == offset + count)
*txq = offset;
if (*txq == res.offset + res.count)
*txq = res.offset;
}
/* Prioritize higher traffic classes, and select among TXQs belonging to the
@@ -1441,15 +1442,14 @@ static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask)
u32 i, queue_mask = 0;
for (i = 0; i < dev->num_tc; i++) {
u32 offset, count;
struct netdev_tc_txq res;
if (!(tc_mask & BIT(i)))
continue;
offset = dev->tc_to_txq[i].offset;
count = dev->tc_to_txq[i].count;
res.combined = READ_ONCE(dev->tc_to_txq[i].combined);
queue_mask |= GENMASK(offset + count - 1, offset);
queue_mask |= GENMASK(res.offset + res.count - 1, res.offset);
}
return queue_mask;
@@ -1802,10 +1802,14 @@ static int taprio_mqprio_cmp(const struct net_device *dev,
if (!mqprio || mqprio->num_tc != dev->num_tc)
return -1;
for (i = 0; i < mqprio->num_tc; i++)
if (dev->tc_to_txq[i].count != mqprio->count[i] ||
dev->tc_to_txq[i].offset != mqprio->offset[i])
for (i = 0; i < mqprio->num_tc; i++) {
struct netdev_tc_txq res;
res.combined = READ_ONCE(dev->tc_to_txq[i].combined);
if (res.count != mqprio->count[i] ||
res.offset != mqprio->offset[i])
return -1;
}
for (i = 0; i <= TC_BITMASK; i++)
if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i])