Skip to content

Commit

Permalink
net: embed num_tc in the xps maps
Browse files Browse the repository at this point in the history
The xps cpus/rxqs map is accessed using dev->num_tc, which is used when
allocating the map. But later updates of dev->num_tc can lead to having
a mismatch between the maps and how they're accessed. In such cases the
map values do not make any sense and out of bound accesses can occur
(that can be easily seen using KASAN).

This patch aims at fixing this by embedding num_tc into the maps, using
the value at the time the map is created. This brings two improvements:
- The maps can be accessed using the embedded num_tc, so we know for
  sure we won't have out of bound accesses.
- Checks can be made before accessing the maps so we know the values
  retrieved will make sense.

We also update __netif_set_xps_queue to conditionally copy old maps from
dev_maps in the new one only if the number of traffic classes from both
maps match.

Signed-off-by: Antoine Tenart <[email protected]>
Signed-off-by: David S. Miller <[email protected]>
  • Loading branch information
atenart authored and davem330 committed Mar 18, 2021
1 parent 73f5e52 commit 255c04a
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 50 deletions.
6 changes: 6 additions & 0 deletions include/linux/netdevice.h
Original file line number Diff line number Diff line change
Expand Up @@ -771,9 +771,15 @@ struct xps_map {

/*
* This structure holds all XPS maps for device. Maps are indexed by CPU.
*
* We keep track of the number of traffic classes used when the struct is
* allocated, in num_tc. This will be used to navigate the maps, to ensure we're
* not crossing its upper bound, as the original dev->num_tc can be updated in
* the meantime.
*/
struct xps_dev_maps {
struct rcu_head rcu;
s16 num_tc;
struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
};

Expand Down
63 changes: 41 additions & 22 deletions net/core/dev.c
Original file line number Diff line number Diff line change
Expand Up @@ -2491,7 +2491,7 @@ static bool remove_xps_queue_cpu(struct net_device *dev,
struct xps_dev_maps *dev_maps,
int cpu, u16 offset, u16 count)
{
int num_tc = dev->num_tc ? : 1;
int num_tc = dev_maps->num_tc;
bool active = false;
int tci;

Expand Down Expand Up @@ -2634,10 +2634,10 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
{
const unsigned long *online_mask = NULL, *possible_mask = NULL;
struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
bool active = false, copy = false;
int i, j, tci, numa_node_id = -2;
int maps_sz, num_tc = 1, tc = 0;
struct xps_map *map, *new_map;
bool active = false;
unsigned int nr_ids;

if (dev->num_tc) {
Expand Down Expand Up @@ -2672,19 +2672,29 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
if (maps_sz < L1_CACHE_BYTES)
maps_sz = L1_CACHE_BYTES;

/* The old dev_maps could be larger or smaller than the one we're
* setting up now, as dev->num_tc could have been updated in between. We
* could try to be smart, but let's be safe instead and only copy
* foreign traffic classes if the two map sizes match.
*/
if (dev_maps && dev_maps->num_tc == num_tc)
copy = true;

/* allocate memory for queue storage */
for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
j < nr_ids;) {
if (!new_dev_maps)
new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
if (!new_dev_maps) {
mutex_unlock(&xps_map_mutex);
return -ENOMEM;
new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
if (!new_dev_maps) {
mutex_unlock(&xps_map_mutex);
return -ENOMEM;
}

new_dev_maps->num_tc = num_tc;
}

tci = j * num_tc + tc;
map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
NULL;
map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;

map = expand_xps_map(map, j, index, is_rxqs_map);
if (!map)
Expand All @@ -2706,7 +2716,7 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
j < nr_ids;) {
/* copy maps belonging to foreign traffic classes */
for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
for (i = tc, tci = j * num_tc; copy && i--; tci++) {
/* fill in the new device map from the old device map */
map = xmap_dereference(dev_maps->attr_map[tci]);
RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Expand Down Expand Up @@ -2736,14 +2746,14 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
numa_node_id = -1;
}
#endif
} else if (dev_maps) {
} else if (copy) {
/* fill in the new device map from the old device map */
map = xmap_dereference(dev_maps->attr_map[tci]);
RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
}

/* copy maps belonging to foreign traffic classes */
for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
for (i = num_tc - tc, tci++; copy && --i; tci++) {
/* fill in the new device map from the old device map */
map = xmap_dereference(dev_maps->attr_map[tci]);
RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Expand All @@ -2761,11 +2771,18 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,

for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
j < nr_ids;) {
for (i = num_tc, tci = j * num_tc; i--; tci++) {
new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
map = xmap_dereference(dev_maps->attr_map[tci]);
if (map && map != new_map)
kfree_rcu(map, rcu);
if (!map)
continue;

if (copy) {
new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
if (map == new_map)
continue;
}

kfree_rcu(map, rcu);
}
}

Expand All @@ -2789,12 +2806,12 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
/* removes tx-queue from unused CPUs/rx-queues */
for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
j < nr_ids;) {
for (i = tc, tci = j * num_tc; i--; tci++)
for (i = tc, tci = j * dev_maps->num_tc; i--; tci++)
active |= remove_xps_queue(dev_maps, tci, index);
if (!netif_attr_test_mask(j, mask, nr_ids) ||
!netif_attr_test_online(j, online_mask, nr_ids))
active |= remove_xps_queue(dev_maps, tci, index);
for (i = num_tc - tc, tci++; --i; tci++)
for (i = dev_maps->num_tc - tc, tci++; --i; tci++)
active |= remove_xps_queue(dev_maps, tci, index);
}

Expand All @@ -2812,7 +2829,7 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
j < nr_ids;) {
for (i = num_tc, tci = j * num_tc; i--; tci++) {
new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
map = dev_maps ?
map = copy ?
xmap_dereference(dev_maps->attr_map[tci]) :
NULL;
if (new_map && new_map != map)
Expand Down Expand Up @@ -3944,13 +3961,15 @@ sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
struct xps_dev_maps *dev_maps, unsigned int tci)
{
int tc = netdev_get_prio_tc_map(dev, skb->priority);
struct xps_map *map;
int queue_index = -1;

if (dev->num_tc) {
tci *= dev->num_tc;
tci += netdev_get_prio_tc_map(dev, skb->priority);
}
if (tc >= dev_maps->num_tc)
return queue_index;

tci *= dev_maps->num_tc;
tci += tc;

map = rcu_dereference(dev_maps->attr_map[tci]);
if (map) {
Expand Down
45 changes: 17 additions & 28 deletions net/core/net-sysfs.c
Original file line number Diff line number Diff line change
Expand Up @@ -1364,9 +1364,9 @@ static const struct attribute_group dql_group = {
static ssize_t xps_cpus_show(struct netdev_queue *queue,
char *buf)
{
int j, len, ret, num_tc = 1, tc = 0;
struct net_device *dev = queue->dev;
struct xps_dev_maps *dev_maps;
int j, len, ret, tc = 0;
unsigned long *mask;
unsigned int index;

Expand All @@ -1378,22 +1378,13 @@ static ssize_t xps_cpus_show(struct netdev_queue *queue,
if (!rtnl_trylock())
return restart_syscall();

if (dev->num_tc) {
/* Do not allow XPS on subordinate device directly */
num_tc = dev->num_tc;
if (num_tc < 0) {
ret = -EINVAL;
goto err_rtnl_unlock;
}

/* If queue belongs to subordinate dev use its map */
dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
/* If queue belongs to subordinate dev use its map */
dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;

tc = netdev_txq_to_tc(dev, index);
if (tc < 0) {
ret = -EINVAL;
goto err_rtnl_unlock;
}
tc = netdev_txq_to_tc(dev, index);
if (tc < 0) {
ret = -EINVAL;
goto err_rtnl_unlock;
}

mask = bitmap_zalloc(nr_cpu_ids, GFP_KERNEL);
Expand All @@ -1404,12 +1395,12 @@ static ssize_t xps_cpus_show(struct netdev_queue *queue,

rcu_read_lock();
dev_maps = rcu_dereference(dev->xps_cpus_map);
if (!dev_maps)
if (!dev_maps || tc >= dev_maps->num_tc)
goto out_no_maps;

for (j = -1; j = netif_attrmask_next(j, NULL, nr_cpu_ids),
j < nr_cpu_ids;) {
int i, tci = j * num_tc + tc;
int i, tci = j * dev_maps->num_tc + tc;
struct xps_map *map;

map = rcu_dereference(dev_maps->attr_map[tci]);
Expand Down Expand Up @@ -1480,9 +1471,9 @@ static struct netdev_queue_attribute xps_cpus_attribute __ro_after_init

static ssize_t xps_rxqs_show(struct netdev_queue *queue, char *buf)
{
int j, len, ret, num_tc = 1, tc = 0;
struct net_device *dev = queue->dev;
struct xps_dev_maps *dev_maps;
int j, len, ret, tc = 0;
unsigned long *mask;
unsigned int index;

Expand All @@ -1491,14 +1482,12 @@ static ssize_t xps_rxqs_show(struct netdev_queue *queue, char *buf)
if (!rtnl_trylock())
return restart_syscall();

if (dev->num_tc) {
num_tc = dev->num_tc;
tc = netdev_txq_to_tc(dev, index);
if (tc < 0) {
ret = -EINVAL;
goto err_rtnl_unlock;
}
tc = netdev_txq_to_tc(dev, index);
if (tc < 0) {
ret = -EINVAL;
goto err_rtnl_unlock;
}

mask = bitmap_zalloc(dev->num_rx_queues, GFP_KERNEL);
if (!mask) {
ret = -ENOMEM;
Expand All @@ -1507,12 +1496,12 @@ static ssize_t xps_rxqs_show(struct netdev_queue *queue, char *buf)

rcu_read_lock();
dev_maps = rcu_dereference(dev->xps_rxqs_map);
if (!dev_maps)
if (!dev_maps || tc >= dev_maps->num_tc)
goto out_no_maps;

for (j = -1; j = netif_attrmask_next(j, NULL, dev->num_rx_queues),
j < dev->num_rx_queues;) {
int i, tci = j * num_tc + tc;
int i, tci = j * dev_maps->num_tc + tc;
struct xps_map *map;

map = rcu_dereference(dev_maps->attr_map[tci]);
Expand Down

0 comments on commit 255c04a

Please sign in to comment.