Skip to content

Commit

Permalink
net/mlx5e: Support adaptive RX coalescing
Browse files Browse the repository at this point in the history 
Striving for high message rate and low interrupt rate.

Usage:
        ethtool -C <interface> adaptive-rx on/off

Signed-off-by: Gil Rockah <[email protected]>
Signed-off-by: Achiad Shochat <[email protected]>
Signed-off-by: Saeed Mahameed <[email protected]>
CC: Arnd Bergmann <[email protected]>
Signed-off-by: David S. Miller <[email protected]>
  • Loading branch information
@gilr8 @davem330
gilr8 authored and davem330 committed Jun 27, 2016
1 parent 9908aa2 commit cb3c7fd
Show file tree
Hide file tree
Showing 6 changed files with 416 additions and 8 deletions.
3 changes: 2 additions & 1 deletion drivers/net/ethernet/mellanox/mlx5/core/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \

mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o \
en_main.o en_fs.o en_ethtool.o en_tx.o en_rx.o \
en_txrx.o en_clock.o vxlan.o en_tc.o en_arfs.o
en_rx_am.o en_txrx.o en_clock.o vxlan.o en_tc.o \
en_arfs.o

mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o
33 changes: 33 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/en.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ struct mlx5e_params {
#ifdef CONFIG_MLX5_CORE_EN_DCB
struct ieee_ets ets;
#endif
bool rx_am_enabled;
};

struct mlx5e_tstamp {
Expand All @@ -213,13 +214,15 @@ struct mlx5e_tstamp {
enum {
MLX5E_RQ_STATE_POST_WQES_ENABLE,
MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
MLX5E_RQ_STATE_AM,
};

struct mlx5e_cq {
/* data path - accessed per cqe */
struct mlx5_cqwq wq;

/* data path - accessed per napi poll */
u16 event_ctr;
struct napi_struct *napi;
struct mlx5_core_cq mcq;
struct mlx5e_channel *channel;
Expand Down Expand Up @@ -247,6 +250,30 @@ struct mlx5e_dma_info {
dma_addr_t addr;
};

struct mlx5e_rx_am_stats {
int ppms; /* packets per msec */
int epms; /* events per msec */
};

struct mlx5e_rx_am_sample {
ktime_t time;
unsigned int pkt_ctr;
u16 event_ctr;
};

struct mlx5e_rx_am { /* Adaptive Moderation */
u8 state;
struct mlx5e_rx_am_stats prev_stats;
struct mlx5e_rx_am_sample start_sample;
struct work_struct work;
u8 profile_ix;
u8 mode;
u8 tune_state;
u8 steps_right;
u8 steps_left;
u8 tired;
};

struct mlx5e_rq {
/* data path */
struct mlx5_wq_ll wq;
Expand All @@ -267,6 +294,8 @@ struct mlx5e_rq {
unsigned long state;
int ix;

struct mlx5e_rx_am am; /* Adaptive Moderation */

/* control */
struct mlx5_wq_ctrl wq_ctrl;
u8 wq_type;
Expand Down Expand Up @@ -637,6 +666,10 @@ void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
struct mlx5e_mpw_info *wi);
struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);

void mlx5e_rx_am(struct mlx5e_rq *rq);
void mlx5e_rx_am_work(struct work_struct *work);
struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode);

void mlx5e_update_stats(struct mlx5e_priv *priv);

int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
Expand Down
18 changes: 16 additions & 2 deletions drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ static int mlx5e_get_coalesce(struct net_device *netdev,
coal->rx_max_coalesced_frames = priv->params.rx_cq_moderation.pkts;
coal->tx_coalesce_usecs = priv->params.tx_cq_moderation.usec;
coal->tx_max_coalesced_frames = priv->params.tx_cq_moderation.pkts;
coal->use_adaptive_rx_coalesce = priv->params.rx_am_enabled;

return 0;
}
Expand All @@ -538,6 +539,10 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_channel *c;
bool restart =
!!coal->use_adaptive_rx_coalesce != priv->params.rx_am_enabled;
bool was_opened;
int err = 0;
int tc;
int i;

Expand All @@ -546,12 +551,18 @@ static int mlx5e_set_coalesce(struct net_device *netdev,

mutex_lock(&priv->state_lock);

was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened && restart) {
mlx5e_close_locked(netdev);
priv->params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce;
}

priv->params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
priv->params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
priv->params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
priv->params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;

if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
if (!was_opened || restart)
goto out;

for (i = 0; i < priv->params.num_channels; ++i) {
Expand All @@ -570,8 +581,11 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
}

out:
if (was_opened && restart)
err = mlx5e_open_locked(netdev);

mutex_unlock(&priv->state_lock);
return 0;
return err;
}

static u32 ptys2ethtool_supported_link(u32 eth_proto_cap)
Expand Down
30 changes: 25 additions & 5 deletions drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@
#include "vxlan.h"

struct mlx5e_rq_param {
u32 rqc[MLX5_ST_SZ_DW(rqc)];
struct mlx5_wq_param wq;
u32 rqc[MLX5_ST_SZ_DW(rqc)];
struct mlx5_wq_param wq;
bool am_enabled;
};

struct mlx5e_sq_param {
Expand Down Expand Up @@ -337,6 +338,9 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
wqe->data.byte_count = cpu_to_be32(byte_count);
}

INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
rq->am.mode = priv->params.rx_cq_period_mode;

rq->wq_type = priv->params.rq_wq_type;
rq->pdev = c->pdev;
rq->netdev = c->netdev;
Expand Down Expand Up @@ -509,6 +513,9 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
if (err)
goto err_disable_rq;

if (param->am_enabled)
set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);

set_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);

sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;
Expand Down Expand Up @@ -537,6 +544,8 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq)
/* avoid destroying rq before mlx5e_poll_rx_cq() is done with it */
napi_synchronize(&rq->channel->napi);

cancel_work_sync(&rq->am.work);

mlx5e_disable_rq(rq);
mlx5e_destroy_rq(rq);
}
Expand Down Expand Up @@ -1112,6 +1121,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
{
struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
struct net_device *netdev = priv->netdev;
struct mlx5e_cq_moder rx_cq_profile;
int cpu = mlx5e_get_cpu(priv, ix);
struct mlx5e_channel *c;
struct mlx5e_sq *sq;
Expand All @@ -1130,6 +1140,11 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
c->mkey_be = cpu_to_be32(priv->mkey.key);
c->num_tc = priv->params.num_tc;

if (priv->params.rx_am_enabled)
rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
else
rx_cq_profile = priv->params.rx_cq_moderation;

mlx5e_build_channeltc_to_txq_map(priv, ix);

netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
Expand All @@ -1143,7 +1158,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
goto err_close_icosq_cq;

err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
priv->params.rx_cq_moderation);
rx_cq_profile);
if (err)
goto err_close_tx_cqs;

Expand Down Expand Up @@ -1243,6 +1258,8 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,

param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
param->wq.linear = 1;

param->am_enabled = priv->params.rx_am_enabled;
}

static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
Expand Down Expand Up @@ -2883,6 +2900,9 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
struct mlx5e_priv *priv = netdev_priv(netdev);
u32 link_speed = 0;
u32 pci_bw = 0;
u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;

priv->params.log_sq_size =
MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
Expand Down Expand Up @@ -2929,8 +2949,8 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
BIT(priv->params.log_rq_size));

mlx5e_set_rx_cq_mode_params(&priv->params,
MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);

priv->params.tx_cq_moderation.usec =
MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
Expand Down
Loading

0 comments on commit cb3c7fd

Please sign in to comment.