Skip to content

Commit

Permalink
netdev: Globally track port status changes
Browse files Browse the repository at this point in the history
Previously, we tracked status changes for ofports on a per-device basis.
Each time in the main thread's loop, we would inspect every ofport
to determine whether the status had changed for corresponding devices.

This patch replaces the per-netdev change_seq with a global 'struct seq'
which tracks status change for all ports. In the average case where
ports are not constantly going up or down, this allows us to check the
sequence once per main loop and not poll any ports. In the worst case,
execution is expected to be similar to how it is currently.

In a test environment of 5000 internal ports and 50 tunnel ports with
bfd, this reduces average CPU usage of the main thread from about 40% to
about 35%.

Signed-off-by: Joe Stringer <[email protected]>
Signed-off-by: Ethan Jackson <[email protected]>
Acked-by: Ethan Jackson <[email protected]>
  • Loading branch information
joestringer authored and ejj committed Dec 12, 2013
1 parent 2ffe604 commit da4a619
Show file tree
Hide file tree
Showing 14 changed files with 140 additions and 171 deletions.
2 changes: 2 additions & 0 deletions lib/automake.mk
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ lib_libopenvswitch_a_SOURCES = \
lib/command-line.c \
lib/command-line.h \
lib/compiler.h \
lib/connectivity.c \
lib/connectivity.h \
lib/coverage.c \
lib/coverage.h \
lib/crc32c.c \
Expand Down
43 changes: 43 additions & 0 deletions lib/connectivity.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright (c) 2013 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include <config.h>
#include "connectivity.h"
#include "ovs-thread.h"
#include "seq.h"

static struct seq *connectivity_seq;

/* Provides a global seq for connectivity changes.
*
* Connectivity monitoring modules should call seq_change() on the returned
* object whenever the status of a port changes, whether the cause is local or
* remote.
*
* Clients can seq_wait() on this object for changes to netdev flags, features,
* ethernet addresses, carrier changes, and bfd/cfm/lacp/stp status. */
struct seq *
connectivity_seq_get(void)
{
static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;

if (ovsthread_once_start(&once)) {
connectivity_seq = seq_create();
ovsthread_once_done(&once);
}

return connectivity_seq;
}
25 changes: 25 additions & 0 deletions lib/connectivity.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright (c) 2013 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef CONNECTIVITY_H
#define CONNECTIVITY_H 1

#include <stdint.h>

/* For tracking connectivity changes globally. */
struct seq *connectivity_seq_get(void);

#endif /* connectivity.h */
39 changes: 8 additions & 31 deletions lib/netdev-bsd.c
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#endif

#include "rtbsd.h"
#include "connectivity.h"
#include "coverage.h"
#include "dynamic-string.h"
#include "fatal-signal.h"
Expand All @@ -55,8 +56,9 @@
#include "ovs-thread.h"
#include "packets.h"
#include "poll-loop.h"
#include "socket-util.h"
#include "seq.h"
#include "shash.h"
#include "socket-util.h"
#include "svec.h"
#include "util.h"
#include "vlog.h"
Expand Down Expand Up @@ -86,7 +88,6 @@ struct netdev_bsd {
struct ovs_mutex mutex;

unsigned int cache_valid;
unsigned int change_seq;

int ifindex;
uint8_t etheraddr[ETH_ADDR_LEN];
Expand Down Expand Up @@ -197,15 +198,6 @@ netdev_bsd_wait(void)
rtbsd_notifier_wait();
}

static void
netdev_bsd_changed(struct netdev_bsd *dev)
{
dev->change_seq++;
if (!dev->change_seq) {
dev->change_seq++;
}
}

/* Invalidate cache in case of interface status change. */
static void
netdev_bsd_cache_cb(const struct rtbsd_change *change,
Expand All @@ -223,7 +215,7 @@ netdev_bsd_cache_cb(const struct rtbsd_change *change,
if (is_netdev_bsd_class(netdev_class)) {
dev = netdev_bsd_cast(base_dev);
dev->cache_valid = 0;
netdev_bsd_changed(dev);
seq_change(connectivity_seq_get());
}
netdev_close(base_dev);
}
Expand All @@ -241,7 +233,7 @@ netdev_bsd_cache_cb(const struct rtbsd_change *change,
struct netdev *netdev = node->data;
dev = netdev_bsd_cast(netdev);
dev->cache_valid = 0;
netdev_bsd_changed(dev);
seq_change(connectivity_seq_get());
netdev_close(netdev);
}
shash_destroy(&device_shash);
Expand Down Expand Up @@ -294,7 +286,6 @@ netdev_bsd_construct_system(struct netdev *netdev_)
}

ovs_mutex_init(&netdev->mutex);
netdev->change_seq = 1;
netdev->tap_fd = -1;
netdev->kernel_name = xstrdup(netdev_->name);

Expand Down Expand Up @@ -329,7 +320,6 @@ netdev_bsd_construct_tap(struct netdev *netdev_)
* to retrieve the name of the tap device. */
ovs_mutex_init(&netdev->mutex);
netdev->tap_fd = open("/dev/tap", O_RDWR);
netdev->change_seq = 1;
if (netdev->tap_fd < 0) {
error = errno;
VLOG_WARN("opening \"/dev/tap\" failed: %s", ovs_strerror(error));
Expand Down Expand Up @@ -506,9 +496,6 @@ netdev_bsd_rx_construct(struct netdev_rx *rx_)
ovs_mutex_lock(&netdev->mutex);
error = netdev_bsd_open_pcap(netdev_get_kernel_name(netdev_),
&rx->pcap_handle, &rx->fd);
if (!error) {
netdev_bsd_changed(netdev);
}
ovs_mutex_unlock(&netdev->mutex);
}

Expand Down Expand Up @@ -756,7 +743,7 @@ netdev_bsd_set_etheraddr(struct netdev *netdev_,
if (!error) {
netdev->cache_valid |= VALID_ETHERADDR;
memcpy(netdev->etheraddr, mac, ETH_ADDR_LEN);
netdev_bsd_changed(netdev);
seq_change(connectivity_seq_get());
}
}
ovs_mutex_unlock(&netdev->mutex);
Expand Down Expand Up @@ -1165,7 +1152,7 @@ netdev_bsd_set_in4(struct netdev *netdev_, struct in_addr addr,
netdev->netmask = mask;
}
}
netdev_bsd_changed(netdev);
seq_change(connectivity_seq_get());
}
ovs_mutex_unlock(&netdev->mutex);

Expand Down Expand Up @@ -1464,18 +1451,12 @@ netdev_bsd_update_flags(struct netdev *netdev_, enum netdev_flags off,
new_flags = (old_flags & ~nd_to_iff_flags(off)) | nd_to_iff_flags(on);
if (new_flags != old_flags) {
error = set_flags(netdev_get_kernel_name(netdev_), new_flags);
netdev_bsd_changed(netdev);
seq_change(connectivity_seq_get());
}
}
return error;
}

static unsigned int
netdev_bsd_change_seq(const struct netdev *netdev)
{
return netdev_bsd_cast(netdev)->change_seq;
}


const struct netdev_class netdev_bsd_class = {
"system",
Expand Down Expand Up @@ -1531,8 +1512,6 @@ const struct netdev_class netdev_bsd_class = {

netdev_bsd_update_flags,

netdev_bsd_change_seq,

netdev_bsd_rx_alloc,
netdev_bsd_rx_construct,
netdev_bsd_rx_destruct,
Expand Down Expand Up @@ -1596,8 +1575,6 @@ const struct netdev_class netdev_tap_class = {

netdev_bsd_update_flags,

netdev_bsd_change_seq,

netdev_bsd_rx_alloc,
netdev_bsd_rx_construct,
netdev_bsd_rx_destruct,
Expand Down
34 changes: 4 additions & 30 deletions lib/netdev-dummy.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

#include <errno.h>

#include "connectivity.h"
#include "flow.h"
#include "list.h"
#include "netdev-provider.h"
Expand All @@ -30,6 +31,7 @@
#include "packets.h"
#include "pcap-file.h"
#include "poll-loop.h"
#include "seq.h"
#include "shash.h"
#include "sset.h"
#include "stream.h"
Expand Down Expand Up @@ -66,7 +68,6 @@ struct netdev_dummy {
int mtu OVS_GUARDED;
struct netdev_stats stats OVS_GUARDED;
enum netdev_flags flags OVS_GUARDED;
unsigned int change_seq OVS_GUARDED;
int ifindex OVS_GUARDED;

struct pstream *pstream OVS_GUARDED;
Expand All @@ -91,8 +92,6 @@ struct netdev_rx_dummy {

static unixctl_cb_func netdev_dummy_set_admin_state;
static int netdev_dummy_construct(struct netdev *);
static void netdev_dummy_changed(struct netdev_dummy *netdev)
OVS_REQUIRES(netdev->mutex);
static void netdev_dummy_queue_packet(struct netdev_dummy *, struct ofpbuf *);

static void dummy_stream_close(struct dummy_stream *);
Expand Down Expand Up @@ -285,7 +284,6 @@ netdev_dummy_construct(struct netdev *netdev_)
netdev->hwaddr[5] = n;
netdev->mtu = 1500;
netdev->flags = 0;
netdev->change_seq = 1;
netdev->ifindex = -EOPNOTSUPP;

netdev->pstream = NULL;
Expand Down Expand Up @@ -571,7 +569,7 @@ netdev_dummy_set_etheraddr(struct netdev *netdev,
ovs_mutex_lock(&dev->mutex);
if (!eth_addr_equals(dev->hwaddr, mac)) {
memcpy(dev->hwaddr, mac, ETH_ADDR_LEN);
netdev_dummy_changed(dev);
seq_change(connectivity_seq_get());
}
ovs_mutex_unlock(&dev->mutex);

Expand Down Expand Up @@ -666,7 +664,7 @@ netdev_dummy_update_flags__(struct netdev_dummy *netdev,
netdev->flags |= on;
netdev->flags &= ~off;
if (*old_flagsp != netdev->flags) {
netdev_dummy_changed(netdev);
seq_change(connectivity_seq_get());
}

return 0;
Expand All @@ -686,31 +684,9 @@ netdev_dummy_update_flags(struct netdev *netdev_,

return error;
}

static unsigned int
netdev_dummy_change_seq(const struct netdev *netdev_)
{
struct netdev_dummy *netdev = netdev_dummy_cast(netdev_);
unsigned int change_seq;

ovs_mutex_lock(&netdev->mutex);
change_seq = netdev->change_seq;
ovs_mutex_unlock(&netdev->mutex);

return change_seq;
}

/* Helper functions. */

static void
netdev_dummy_changed(struct netdev_dummy *dev)
{
dev->change_seq++;
if (!dev->change_seq) {
dev->change_seq++;
}
}

static const struct netdev_class dummy_class = {
"dummy",
NULL, /* init */
Expand Down Expand Up @@ -766,8 +742,6 @@ static const struct netdev_class dummy_class = {

netdev_dummy_update_flags,

netdev_dummy_change_seq,

netdev_dummy_rx_alloc,
netdev_dummy_rx_construct,
netdev_dummy_rx_destruct,
Expand Down
24 changes: 3 additions & 21 deletions lib/netdev-linux.c
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include <string.h>
#include <unistd.h>

#include "connectivity.h"
#include "coverage.h"
#include "dpif-linux.h"
#include "dynamic-string.h"
Expand All @@ -65,6 +66,7 @@
#include "packets.h"
#include "poll-loop.h"
#include "rtnetlink-link.h"
#include "seq.h"
#include "shash.h"
#include "socket-util.h"
#include "sset.h"
Expand Down Expand Up @@ -357,7 +359,6 @@ struct netdev_linux {
struct ovs_mutex mutex;

unsigned int cache_valid;
unsigned int change_seq;

bool miimon; /* Link status of last poll. */
long long int miimon_interval; /* Miimon Poll rate. Disabled if <= 0. */
Expand Down Expand Up @@ -585,10 +586,7 @@ netdev_linux_changed(struct netdev_linux *dev,
unsigned int ifi_flags, unsigned int mask)
OVS_REQUIRES(dev->mutex)
{
dev->change_seq++;
if (!dev->change_seq) {
dev->change_seq++;
}
seq_change(connectivity_seq_get());

if ((dev->ifi_flags ^ ifi_flags) & IFF_RUNNING) {
dev->carrier_resets++;
Expand Down Expand Up @@ -640,7 +638,6 @@ static void
netdev_linux_common_construct(struct netdev_linux *netdev)
{
ovs_mutex_init(&netdev->mutex);
netdev->change_seq = 1;
}

/* Creates system and internal devices. */
Expand Down Expand Up @@ -2597,19 +2594,6 @@ netdev_linux_update_flags(struct netdev *netdev_, enum netdev_flags off,
return error;
}

static unsigned int
netdev_linux_change_seq(const struct netdev *netdev_)
{
struct netdev_linux *netdev = netdev_linux_cast(netdev_);
unsigned int change_seq;

ovs_mutex_lock(&netdev->mutex);
change_seq = netdev->change_seq;
ovs_mutex_unlock(&netdev->mutex);

return change_seq;
}

#define NETDEV_LINUX_CLASS(NAME, CONSTRUCT, GET_STATS, SET_STATS, \
GET_FEATURES, GET_STATUS) \
{ \
Expand Down Expand Up @@ -2668,8 +2652,6 @@ netdev_linux_change_seq(const struct netdev *netdev_)
\
netdev_linux_update_flags, \
\
netdev_linux_change_seq, \
\
netdev_linux_rx_alloc, \
netdev_linux_rx_construct, \
netdev_linux_rx_destruct, \
Expand Down
Loading

0 comments on commit da4a619

Please sign in to comment.