Build on Jeff Roberson's linker-set based dynamic per-CPU allocator

(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator.  Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...).  This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.

Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack.  Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory.  Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.

Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy.  Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address.  When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.

This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.

Bump __FreeBSD_version and update UPDATING.

Portions submitted by:  bz
Reviewed by:            bz, zec
Discussed with:         gnn, jamie, jeff, jhb, julian, sam
Suggested by:           peter
Approved by:            re (kensmith)

ObsoleteFiles.inc

@@ -14,6 +14,10 @@
 # The file is partitioned: OLD_FILES first, then OLD_LIBS and OLD_DIRS last.
 #
 
+# 20090713: vimage container structs removed.
+OLD_FILES+=usr/include/netinet/vinet.h
+OLD_FILES+=usr/include/netinet6/vinet6.h
+OLD_FILES+=usr/include/netipsec/vipsec.h
 # 20090709: msgctl.3 msgget.3 msgrcv.3 msgsnd.3 manual pages moved
 OLD_FILES+=usr/share/man/man3/msgctl.3.gz
 OLD_FILES+=usr/share/man/man3/msgget.3.gz

UPDATING

@@ -22,6 +22,11 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 8.x IS SLOW:
 	to maximize performance.  (To disable malloc debugging, run
 	ln -s aj /etc/malloc.conf.)
 
+20090714:
+	Due to changes in the implementation of virtual network stack
+	support, all network-related kernel modules must be recompiled.
+	As this change breaks the ABI, bump __FreeBSD_version to 800104.
+
 20090713:
 	The TOE interface to the TCP syncache has been modified to remove struct
 	tcpopt (<netinet/tcp_var.h>) from the ABI of the network stack.  The

sys/compat/linprocfs/linprocfs.c

@@ -1078,7 +1078,6 @@ linprocfs_doprocmaps(PFS_FILL_ARGS)
 static int
 linprocfs_donetdev(PFS_FILL_ARGS)
 {
-	INIT_VNET_NET(TD_TO_VNET(td));
 	char ifname[16]; /* XXX LINUX_IFNAMSIZ */
 	struct ifnet *ifp;
 

sys/compat/linux/linux_ioctl.c

@@ -2059,7 +2059,6 @@ linux_ioctl_console(struct thread *td, struct linux_ioctl_args *args)
 int
 linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen)
 {
-	INIT_VNET_NET(ifp->if_vnet);
 	struct ifnet *ifscan;
 	int ethno;
 
@@ -2093,7 +2092,6 @@ linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen)
 static struct ifnet *
 ifname_linux_to_bsd(struct thread *td, const char *lxname, char *bsdname)
 {
-	INIT_VNET_NET(TD_TO_VNET(td));
 	struct ifnet *ifp;
 	int len, unit;
 	char *ep;
@@ -2134,7 +2132,6 @@ ifname_linux_to_bsd(struct thread *td, const char *lxname, char *bsdname)
 static int
 linux_ifconf(struct thread *td, struct ifconf *uifc)
 {
-	INIT_VNET_NET(TD_TO_VNET(td));
 #ifdef COMPAT_LINUX32
 	struct l_ifconf ifc;
 #else

sys/compat/linux/linux_socket.c

@@ -60,7 +60,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/ip6.h>
 #include <netinet6/ip6_var.h>
 #include <netinet6/in6_var.h>
-#include <netinet6/vinet6.h>
 #endif
 
 #ifdef COMPAT_LINUX32
@@ -599,11 +598,6 @@ struct linux_socket_args {
 static int
 linux_socket(struct thread *td, struct linux_socket_args *args)
 {
-#ifdef INET6
-#ifndef KLD_MODULE
-	INIT_VNET_INET6(curvnet);
-#endif
-#endif
 	struct socket_args /* {
 		int domain;
 		int type;

sys/compat/svr4/svr4_sockio.c

@@ -89,7 +89,6 @@ svr4_sock_ioctl(fp, td, retval, fd, cmd, data)
 	switch (cmd) {
 	case SVR4_SIOCGIFNUM:
 		{
-			INIT_VNET_NET(curvnet);
 			struct ifnet *ifp;
 			struct ifaddr *ifa;
 			int ifnum = 0;

sys/conf/files

@@ -1989,7 +1989,7 @@ kern/kern_time.c		standard
 kern/kern_timeout.c		standard
 kern/kern_umtx.c		standard
 kern/kern_uuid.c		standard
-kern/kern_vimage.c		standard
+kern/kern_vimage.c		optional vimage
 kern/kern_xxx.c			standard
 kern/link_elf.c			standard
 kern/linker_if.m		standard
@@ -2231,6 +2231,7 @@ net/route.c			standard
 net/rtsock.c			standard
 net/slcompress.c		optional netgraph_vjc | sppp | \
 					 netgraph_sppp
+net/vnet.c			optional vimage
 net/zlib.c			optional crypto | geom_uzip | ipsec | \
 					 mxge | netgraph_deflate | \
 					 ddb_ctf

sys/conf/options

@@ -823,7 +823,6 @@ TDMA_TXRATE_11NG_DEFAULT	opt_tdma.h
 
 # Virtualize the network stack
 VIMAGE			opt_global.h
-VIMAGE_GLOBALS		opt_global.h
 
 # Common Flash Interface (CFI) options
 CFI_SUPPORT_STRATAFLASH	opt_cfi.h

sys/contrib/altq/altq/altq_subr.c

@@ -469,7 +469,6 @@ tbr_timeout(arg)
 	VNET_LIST_RLOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
-		INIT_VNET_NET(vnet_iter);
 #endif
 		for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 		    ifp = TAILQ_NEXT(ifp, if_list)) {

sys/contrib/ipfilter/netinet/ip_fil_freebsd.c

@@ -64,6 +64,7 @@ static const char rcsid[] = "@(#)$Id: ip_fil_freebsd.c,v 2.53.2.50 2007/09/20 12
 #endif
 #if __FreeBSD_version >= 800044
 # include <sys/vimage.h>
+# include <netinet/tcp_var.h>
 #else
 #define V_path_mtu_discovery path_mtu_discovery
 #define V_ipforwarding ipforwarding
@@ -92,9 +93,6 @@ static const char rcsid[] = "@(#)$Id: ip_fil_freebsd.c,v 2.53.2.50 2007/09/20 12
 #include <netinet/udp.h>
 #include <netinet/tcpip.h>
 #include <netinet/ip_icmp.h>
-#if defined(__FreeBSD_version) && (__FreeBSD_version >= 800056)
-# include <netinet/vinet.h>
-#endif
 #ifndef _KERNEL
 # include "netinet/ipf.h"
 #endif
@@ -213,7 +211,6 @@ char *s;
 
 int ipfattach()
 {
-	INIT_VNET_INET(curvnet);
 #ifdef USE_SPL
 	int s;
 #endif
@@ -265,7 +262,6 @@ int ipfattach()
  */
 int ipfdetach()
 {
-	INIT_VNET_INET(curvnet);
 #ifdef USE_SPL
 	int s;
 #endif
@@ -654,7 +650,6 @@ static int fr_send_ip(fin, m, mpp)
 fr_info_t *fin;
 mb_t *m, **mpp;
 {
-	INIT_VNET_INET(curvnet);
 	fr_info_t fnew;
 	ip_t *ip, *oip;
 	int hlen;

sys/contrib/pf/net/pf.c

@@ -119,9 +119,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/udp_var.h>
 #include <netinet/icmp_var.h>
 #include <netinet/if_ether.h>
-#ifdef __FreeBSD__
-#include <netinet/vinet.h>
-#endif
 
 #ifndef __FreeBSD__
 #include <dev/rndvar.h>
@@ -141,7 +138,6 @@ __FBSDID("$FreeBSD$");
 #ifdef __FreeBSD__
 #include <netinet6/ip6_var.h>
 #include <netinet6/in6_pcb.h>
-#include <netinet6/vinet6.h>
 #endif
 #endif /* INET6 */
 
@@ -1762,7 +1758,6 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
     u_int8_t flags, u_int16_t win, u_int16_t mss, u_int8_t ttl, int tag,
     u_int16_t rtag, struct ether_header *eh, struct ifnet *ifp)
 {
-	INIT_VNET_INET(curvnet);
 	struct mbuf	*m;
 	int		 len, tlen;
 #ifdef INET
@@ -2926,7 +2921,6 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd, struct inpcb *inp_arg)
 pf_socket_lookup(int direction, struct pf_pdesc *pd)
 #endif
 {
-	INIT_VNET_INET(curvnet);
 	struct pf_addr		*saddr, *daddr;
 	u_int16_t		 sport, dport;
 #ifdef __FreeBSD__
@@ -3096,7 +3090,6 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
 u_int16_t
 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
 {
-	INIT_VNET_INET(curvnet);
 	int		 hlen;
 	u_int8_t	 hdr[60];
 	u_int8_t	*opt, optlen;
@@ -3136,7 +3129,6 @@ u_int16_t
 pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer)
 {
 #ifdef INET
-	INIT_VNET_INET(curvnet);
 	struct sockaddr_in	*dst;
 	struct route		 ro;
 #endif /* INET */
@@ -3239,7 +3231,6 @@ pf_test_tcp(struct pf_rule **rm, struct pf_state **sm, int direction,
     struct ifqueue *ifq)
 #endif
 {
-	INIT_VNET_INET(curvnet);
 	struct pf_rule		*nr = NULL;
 	struct pf_addr		*saddr = pd->src, *daddr = pd->dst;
 	struct tcphdr		*th = pd->hdr.tcp;
@@ -6094,7 +6085,6 @@ void
 pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
     struct pf_state *s, struct pf_pdesc *pd)
 {
-	INIT_VNET_INET(curvnet);
 	struct mbuf		*m0, *m1;
 	struct route		 iproute;
 	struct route		*ro = NULL;
@@ -6633,26 +6623,22 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t a
 		switch (p) {
 		case IPPROTO_TCP:
 		    {
-			INIT_VNET_INET(curvnet);
 			TCPSTAT_INC(tcps_rcvbadsum);
 			break;
 		    }
 		case IPPROTO_UDP:
 		    {
-			INIT_VNET_INET(curvnet);
 			UDPSTAT_INC(udps_badsum);
 			break;
 		    }
 		case IPPROTO_ICMP:
 		    {
-			INIT_VNET_INET(curvnet);
 			ICMPSTAT_INC(icps_checksum);
 			break;
 		    }
 #ifdef INET6
 		case IPPROTO_ICMPV6:
 		    {
-			INIT_VNET_INET6(curvnet);
 			ICMP6STAT_INC(icp6s_checksum);
 			break;
 		    }

sys/contrib/pf/net/pf_if.c

@@ -115,9 +115,6 @@ void		 pfi_change_group_event(void * __unused, char *);
 void		 pfi_detach_group_event(void * __unused, struct ifg_group *);
 void		 pfi_ifaddr_event(void * __unused, struct ifnet *);
 
-#ifdef VIMAGE_GLOBALS
-extern struct ifgrouphead ifg_head;
-#endif
 #endif
 
 RB_PROTOTYPE(pfi_ifhead, pfi_kif, pfik_tree, pfi_if_compare);
@@ -129,7 +126,6 @@ RB_GENERATE(pfi_ifhead, pfi_kif, pfik_tree, pfi_if_compare);
 void
 pfi_initialize(void)
 {
-	INIT_VNET_NET(curvnet);
 
 	if (pfi_all != NULL)	/* already initialized */
 		return;

sys/contrib/pf/net/pf_ioctl.c

@@ -3703,7 +3703,6 @@ static int
 pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir,
     struct inpcb *inp)
 {
-	INIT_VNET_NET(curvnet);
 
 	/*
 	 * IPv6 is not affected by ip_len/ip_off byte order changes.

sys/contrib/pf/net/pf_subr.c

@@ -65,7 +65,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_timer.h>
 #include <netinet/tcp_var.h>
 #include <netinet/if_ether.h>
-#include <netinet/vinet.h>
 #include <net/pfvar.h>
 
 /*

sys/contrib/pf/net/pfvar.h

@@ -1855,12 +1855,4 @@ int	pf_osfp_match(struct pf_osfp_enlist *, pf_osfp_t);
 struct pf_os_fingerprint *
 	pf_osfp_validate(void);
 
-/*
- * Symbol translation macros
- */
-#define	INIT_VNET_PF(vnet) \
-	INIT_FROM_VNET(vnet, VNET_MOD_PF, struct vnet_pf, vnet_pf)
-
-#define	VNET_PF(sym)	VSYM(vnet_pf, sym)
-
 #endif /* _NET_PFVAR_H_ */

sys/contrib/rdma/rdma_cma.c

@@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
-#include <netinet/vinet.h>
 
 #include <contrib/rdma/rdma_cm.h>
 #include <contrib/rdma/ib_cache.h>
@@ -1950,7 +1949,6 @@ static int cma_alloc_port(struct kvl *ps, struct rdma_id_private *id_priv,
 
 static int cma_alloc_any_port(struct kvl *ps, struct rdma_id_private *id_priv)
 {
-	INIT_VNET_INET(curvnet);
 	struct rdma_bind_list *bind_list;
 	int port, ret;
 
@@ -1995,7 +1993,6 @@ static int cma_alloc_any_port(struct kvl *ps, struct rdma_id_private *id_priv)
 
 static int cma_use_port(struct kvl *ps, struct rdma_id_private *id_priv)
 {
-	INIT_VNET_INET(curvnet);
 	struct rdma_id_private *cur_id;
 	struct sockaddr_in *sin, *cur_sin;
 	struct rdma_bind_list *bind_list;
@@ -2915,7 +2912,6 @@ static void cma_remove_one(struct ib_device *device)
 
 static int cma_init(void)
 {
-	INIT_VNET_INET(curvnet);
 	int ret;
 
 	LIST_INIT(&listen_any_list);

sys/ddb/db_expr.c

@@ -51,7 +51,9 @@ db_term(db_expr_t *valuep)
 
 	t = db_read_token();
 	if (t == tIDENT) {
-	    if (!db_value_of_name(db_tok_string, valuep)) {
+	    if (!db_value_of_name(db_tok_string, valuep) &&
+		!db_value_of_name_pcpu(db_tok_string, valuep) &&
+		!db_value_of_name_vnet(db_tok_string, valuep)) {
 		db_error("Symbol not found\n");
 		/*NOTREACHED*/
 	    }