Skip to content

Commit

Permalink
dpif-netdev: Add percentage of pmd/core used by each rxq.
Browse files Browse the repository at this point in the history 
It is based on the length of history that is stored about an
rxq (currently 1 min).

$ ovs-appctl dpif-netdev/pmd-rxq-show
pmd thread numa_id 0 core_id 4:
        isolated : false
        port: dpdkphy1         queue-id:  0    pmd usage: 70 %
        port: dpdkvhost0       queue-id:  0    pmd usage:  0 %
pmd thread numa_id 0 core_id 6:
        isolated : false
        port: dpdkphy0         queue-id:  0    pmd usage: 64 %
        port: dpdkvhost1       queue-id:  0    pmd usage:  0 %

These values are what would be used as part of rxq to pmd
assignment due to a reconfiguration event e.g. adding pmds,
adding rxqs or with the command:

ovs-appctl dpif-netdev/pmd-rxq-rebalance

Signed-off-by: Jan Scheurich <[email protected]>
Co-authored-by: Jan Scheurich <[email protected]>
Signed-off-by: Kevin Traynor <[email protected]>
Signed-off-by: Ian Stokes <[email protected]>
  • Loading branch information
@kevintraynor @JanScheurich @istokes
2 people authored and istokes committed Jan 17, 2018
1 parent 4f5d13e commit 2a2c67b
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 26 deletions.
11 changes: 11 additions & 0 deletions Documentation/howto/dpdk.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,17 @@ Core 3: Q1 (80%) |
Core 7: Q4 (70%) | Q5 (10%)
core 8: Q3 (60%) | Q0 (30%)

To see the current measured usage history of pmd core cycles for each rxq::

$ ovs-appctl dpif-netdev/pmd-rxq-show

.. note::

A history of one minute is recorded and shown for each rxq to allow for
traffic pattern spikes. An rxq's pmd core cycles usage changes due to traffic
pattern or reconfig changes will take one minute before they are fully
reflected in the stats.

Rxq to pmds assignment takes place whenever there are configuration changes
or can be triggered by using::

Expand Down
1 change: 1 addition & 0 deletions NEWS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ Post-v2.8.0
"management" statistics.
- ovs-ofctl dump-ports command now prints new of set custom statistics
if available (for OpenFlow 1.4+).
* Add rxq utilization of pmd to appctl 'dpif-netdev/pmd-rxq-show'.
- Userspace datapath:
* Output packet batching support.
- vswitchd:
Expand Down
53 changes: 40 additions & 13 deletions lib/dpif-netdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,11 @@ struct dp_netdev_pmd_thread {
are stored for each polled rxq. */
long long int rxq_next_cycle_store;

/* Last interval timestamp. */
uint64_t intrvl_tsc_prev;
/* Last interval cycles. */
atomic_ullong intrvl_cycles;

/* Current context of the PMD thread. */
struct dp_netdev_pmd_thread_ctx ctx;

Expand Down Expand Up @@ -932,32 +937,44 @@ static void
pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd)
{
if (pmd->core_id != NON_PMD_CORE_ID) {
const char *prev_name = NULL;
struct rxq_poll *list;
size_t i, n;
size_t n_rxq;
uint64_t total_cycles = 0;

ds_put_format(reply,
"pmd thread numa_id %d core_id %u:\n\tisolated : %s\n",
pmd->numa_id, pmd->core_id, (pmd->isolated)
? "true" : "false");

ovs_mutex_lock(&pmd->port_mutex);
sorted_poll_list(pmd, &list, &n);
for (i = 0; i < n; i++) {
const char *name = netdev_rxq_get_name(list[i].rxq->rx);
sorted_poll_list(pmd, &list, &n_rxq);

if (!prev_name || strcmp(name, prev_name)) {
if (prev_name) {
ds_put_cstr(reply, "\n");
}
ds_put_format(reply, "\tport: %s\tqueue-id:", name);
/* Get the total pmd cycles for an interval. */
atomic_read_relaxed(&pmd->intrvl_cycles, &total_cycles);
/* Estimate the cycles to cover all intervals. */
total_cycles *= PMD_RXQ_INTERVAL_MAX;

for (int i = 0; i < n_rxq; i++) {
struct dp_netdev_rxq *rxq = list[i].rxq;
const char *name = netdev_rxq_get_name(rxq->rx);
uint64_t proc_cycles = 0;

for (int j = 0; j < PMD_RXQ_INTERVAL_MAX; j++) {
proc_cycles += dp_netdev_rxq_get_intrvl_cycles(rxq, j);
}
ds_put_format(reply, " %d",
ds_put_format(reply, "\tport: %-16s\tqueue-id: %2d", name,
netdev_rxq_get_queue_id(list[i].rxq->rx));
prev_name = name;
ds_put_format(reply, "\tpmd usage: ");
if (total_cycles) {
ds_put_format(reply, "%2"PRIu64"",
proc_cycles * 100 / total_cycles);
ds_put_cstr(reply, " %");
} else {
ds_put_format(reply, "%s", "NOT AVAIL");
}
ds_put_cstr(reply, "\n");
}
ovs_mutex_unlock(&pmd->port_mutex);
ds_put_cstr(reply, "\n");
free(list);
}
}
Expand Down Expand Up @@ -4117,6 +4134,8 @@ pmd_thread_main(void *f_)
lc = UINT_MAX;
}

pmd->intrvl_tsc_prev = 0;
atomic_store_relaxed(&pmd->intrvl_cycles, 0);
cycles_counter_update(s);
for (;;) {
uint64_t iter_packets = 0;
Expand Down Expand Up @@ -6116,6 +6135,7 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd,
struct dpcls *cls;

if (pmd->ctx.now > pmd->rxq_next_cycle_store) {
uint64_t curr_tsc;
/* Get the cycles that were used to process each queue and store. */
for (unsigned i = 0; i < poll_cnt; i++) {
uint64_t rxq_cyc_curr = dp_netdev_rxq_get_cycles(poll_list[i].rxq,
Expand All @@ -6124,6 +6144,13 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd,
dp_netdev_rxq_set_cycles(poll_list[i].rxq, RXQ_CYCLES_PROC_CURR,
0);
}
curr_tsc = cycles_counter_update(&pmd->perf_stats);
if (pmd->intrvl_tsc_prev) {
/* There is a prev timestamp, store a new intrvl cycle count. */
atomic_store_relaxed(&pmd->intrvl_cycles,
curr_tsc - pmd->intrvl_tsc_prev);
}
pmd->intrvl_tsc_prev = curr_tsc;
/* Start new measuring interval */
pmd->rxq_next_cycle_store = pmd->ctx.now + PMD_RXQ_INTERVAL_LEN;
}
Expand Down
51 changes: 38 additions & 13 deletions tests/pmd.at
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@ m4_divert_push([PREPARE_TESTS])
# of every rxq (one per line) in the form:
# port_name rxq_id numa_id core_id
parse_pmd_rxq_show () {
awk '/pmd/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\t/{for (i=4; i<=NF; i++) print $2, $i, numa, core}' | sort
awk '/pmd thread/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\tport:/ {print $2, $4, numa, core}' | sort
}

# Given the output of `ovs-appctl dpif-netdev/pmd-rxq-show`,
# and with queues for each core on one line, prints the rxqs
# of the core on one line
# 'port:' port_name 'queue_id:' rxq_id rxq_id rxq_id rxq_id
parse_pmd_rxq_show_group () {
awk '/port:/ {print $1, $2, $3, $4, $12, $20, $28}'
}

# Given the output of `ovs-appctl dpctl/dump-flows`, prints a list of flows
Expand Down Expand Up @@ -53,7 +61,7 @@ m4_define([CHECK_PMD_THREADS_CREATED], [
])

m4_define([SED_NUMA_CORE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1<cleared>\2<cleared>:/"])
m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1<cleared>\2<cleared>:/;s/\(queue-id: \)1 2 5 6/\1<cleared>/;s/\(queue-id: \)0 3 4 7/\1<cleared>/"])
m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/1 2 5 6/<group>/;s/0 3 4 7/<group>/"])
m4_define([DUMMY_NUMA], [--dummy-numa="0,0,0,0"])

AT_SETUP([PMD - creating a thread/add-port])
Expand All @@ -65,7 +73,7 @@ CHECK_PMD_THREADS_CREATED()
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: 0
port: p0 queue-id: 0 pmd usage: NOT AVAIL
])

AT_CHECK([ovs-appctl dpif/show | sed 's/\(tx_queues=\)[[0-9]]*/\1<cleared>/g'], [0], [dnl
Expand Down Expand Up @@ -96,7 +104,14 @@ dummy@ovs-dummy: hit:0 missed:0
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: 0 1 2 3 4 5 6 7
port: p0 queue-id: 0 pmd usage: NOT AVAIL
port: p0 queue-id: 1 pmd usage: NOT AVAIL
port: p0 queue-id: 2 pmd usage: NOT AVAIL
port: p0 queue-id: 3 pmd usage: NOT AVAIL
port: p0 queue-id: 4 pmd usage: NOT AVAIL
port: p0 queue-id: 5 pmd usage: NOT AVAIL
port: p0 queue-id: 6 pmd usage: NOT AVAIL
port: p0 queue-id: 7 pmd usage: NOT AVAIL
])

OVS_VSWITCHD_STOP
Expand All @@ -120,20 +135,23 @@ dummy@ovs-dummy: hit:0 missed:0
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: 0 1 2 3 4 5 6 7
port: p0 queue-id: 0 pmd usage: NOT AVAIL
port: p0 queue-id: 1 pmd usage: NOT AVAIL
port: p0 queue-id: 2 pmd usage: NOT AVAIL
port: p0 queue-id: 3 pmd usage: NOT AVAIL
port: p0 queue-id: 4 pmd usage: NOT AVAIL
port: p0 queue-id: 5 pmd usage: NOT AVAIL
port: p0 queue-id: 6 pmd usage: NOT AVAIL
port: p0 queue-id: 7 pmd usage: NOT AVAIL
])

TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x3])
CHECK_PMD_THREADS_CREATED([2], [], [+$TMP])

AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: <cleared>
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: <cleared>
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed ':a;/AVAIL$/{N;s/\n//;ba}' | parse_pmd_rxq_show_group | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl
port: p0 queue-id: <group>
port: p0 queue-id: <group>
])

TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
Expand All @@ -143,7 +161,14 @@ CHECK_PMD_THREADS_CREATED([1], [], [+$TMP])
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
pmd thread numa_id <cleared> core_id <cleared>:
isolated : false
port: p0 queue-id: 0 1 2 3 4 5 6 7
port: p0 queue-id: 0 pmd usage: NOT AVAIL
port: p0 queue-id: 1 pmd usage: NOT AVAIL
port: p0 queue-id: 2 pmd usage: NOT AVAIL
port: p0 queue-id: 3 pmd usage: NOT AVAIL
port: p0 queue-id: 4 pmd usage: NOT AVAIL
port: p0 queue-id: 5 pmd usage: NOT AVAIL
port: p0 queue-id: 6 pmd usage: NOT AVAIL
port: p0 queue-id: 7 pmd usage: NOT AVAIL
])

OVS_VSWITCHD_STOP
Expand Down

0 comments on commit 2a2c67b

Please sign in to comment.