Widen NET_EPOCH coverage.

When epoch(9) was introduced to network stack, it was basically
dropped in place of existing locking, which was mutexes and
rwlocks. For the sake of performance mutex covered areas were
as small as possible, so became epoch covered areas.

However, epoch doesn't introduce any contention, it just delays
memory reclaim. So, there is no point to minimise epoch covered
areas in sense of performance. Meanwhile entering/exiting epoch
also has non-zero CPU usage, so doing this less often is a win.

Not the least is also code maintainability. In the new paradigm
we can assume that at any stage of processing a packet, we are
inside network epoch. This makes coding both input and output
path way easier.

On output path we already enter epoch quite early - in the
ip_output(), in the ip6_output().

This patch does the same for the input path. All ISR processing,
network related callouts, other ways of packet injection to the
network stack shall be performed in net_epoch. Any leaf function
that walks network configuration now asserts epoch.

Tricky part is configuration code paths - ioctls, sysctls. They
also call into leaf functions, so some need to be changed.

This patch would introduce more epoch recursions (see EPOCH_TRACE)
than we had before. They will be cleaned up separately, as several
of them aren't trivial. Note, that unlike a lock recursion the
epoch recursion is safe and just wastes a bit of resources.

Reviewed by:	gallatin, hselasky, cy, adrian, kristof
Differential Revision:	https://reviews.freebsd.org/D19111

sys/contrib/ipfilter/netinet/ip_fil_freebsd.c

@@ -1312,15 +1312,18 @@ ipf_inject(fin, m)
 	fr_info_t *fin;
 	mb_t *m;
 {
+	struct epoch_tracker et;
 	int error = 0;
 
+	NET_EPOCH_ENTER(et);
 	if (fin->fin_out == 0) {
 		netisr_dispatch(NETISR_IP, m);
 	} else {
 		fin->fin_ip->ip_len = ntohs(fin->fin_ip->ip_len);
 		fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off);
 		error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL);
 	}
+	NET_EPOCH_EXIT(et);
 
 	return error;
 }

sys/dev/firewire/if_fwip.c

@@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count)
 static void
 fwip_stream_input(struct fw_xferq *xferq)
 {
+	struct epoch_tracker et;
 	struct mbuf *m, *m0;
 	struct m_tag *mtag;
 	struct ifnet *ifp;
@@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq)
 	fwip = (struct fwip_softc *)xferq->sc;
 	ifp = fwip->fw_softc.fwip_ifp;
 
+	NET_EPOCH_ENTER(et);
 	while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) {
 		STAILQ_REMOVE_HEAD(&xferq->stvalid, link);
 		fp = mtod(sxfer->mbuf, struct fw_pkt *);
@@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq)
 		firewire_input(ifp, m, src);
 		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 	}
+	NET_EPOCH_EXIT(et);
 	if (STAILQ_FIRST(&xferq->stfree) != NULL)
 		fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch);
 }

sys/dev/iicbus/if_ic.c

@@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr)
 		BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN);
 		top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0);
 		if (top) {
+			struct epoch_tracker et;
+
 			mtx_unlock(&sc->ic_lock);
 			M_SETFIB(top, sc->ic_ifp->if_fib);
+			NET_EPOCH_ENTER(et);
 			netisr_dispatch(NETISR_IP, top);
+			NET_EPOCH_EXIT(et);
 			mtx_lock(&sc->ic_lock);
 		}
 		break;

sys/dev/usb/net/if_usie.c

@@ -773,6 +773,7 @@ usie_uc_status_callback(struct usb_xfer *xfer, usb_error_t error)
 static void
 usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 {
+	struct epoch_tracker et;
 	struct usie_softc *sc = usbd_xfer_softc(xfer);
 	struct ifnet *ifp = sc->sc_ifp;
 	struct mbuf *m0;
@@ -852,6 +853,7 @@ usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 	err = pkt = 0;
 
 	/* HW can aggregate multiple frames in a single USB xfer */
+	NET_EPOCH_ENTER(et);
 	for (;;) {
 		rxd = mtod(m, struct usie_desc *);
 
@@ -918,6 +920,7 @@ usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 		m->m_data += diff;
 		m->m_pkthdr.len = (m->m_len -= diff);
 	}
+	NET_EPOCH_EXIT(et);
 
 	mtx_lock(&sc->sc_mtx);
 

sys/dev/usb/net/uhso.c

@@ -1664,6 +1664,7 @@ uhso_ifnet_read_callback(struct usb_xfer *xfer, usb_error_t error)
 static void
 uhso_if_rxflush(void *arg)
 {
+	struct epoch_tracker et;
 	struct uhso_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	uint8_t *cp;
@@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg)
 
 	m = NULL;
 	mwait = sc->sc_mwait;
+	NET_EPOCH_ENTER(et);
 	for (;;) {
 		if (m == NULL) {
 			if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL)
@@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg)
 		m = m0 != NULL ? m0 : NULL;
 		mtx_lock(&sc->sc_mtx);
 	}
+	NET_EPOCH_EXIT(et);
 	sc->sc_mwait = mwait;
 }
 

sys/dev/usb/net/usb_ethernet.c

@@ -645,22 +645,21 @@ void
 uether_rxflush(struct usb_ether *ue)
 {
 	struct ifnet *ifp = ue->ue_ifp;
-	struct mbuf *m;
+	struct epoch_tracker et;
+	struct mbuf *m, *n;
 
 	UE_LOCK_ASSERT(ue, MA_OWNED);
 
-	for (;;) {
-		m = mbufq_dequeue(&ue->ue_rxq);
-		if (m == NULL)
-			break;
-
-		/*
-		 * The USB xfer has been resubmitted so its safe to unlock now.
-		 */
-		UE_UNLOCK(ue);
+	n = mbufq_flush(&ue->ue_rxq);
+	UE_UNLOCK(ue);
+	NET_EPOCH_ENTER(et);
+	while ((m = n) != NULL) {
+		n = STAILQ_NEXT(m, m_stailqpkt);
+		m->m_nextpkt = NULL;
 		ifp->if_input(ifp, m);
-		UE_LOCK(ue);
 	}
+	NET_EPOCH_EXIT(et);
+	UE_LOCK(ue);
 }
 
 /*

sys/kern/uipc_socket.c

@@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp.h>
 
 #include <net/vnet.h>
+#include <net/if.h>	/* XXXGL: net_epoch should move out there */
+#include <net/if_var.h>	/* XXXGL: net_epoch should move out there */
 
 #include <security/mac/mac_framework.h>