Continuation of multi page mbuf redesign from r359919.

The following series of patches addresses three things:

Now that array of pages is embedded into mbuf, we no longer need
separate structure to pass around, so struct mbuf_ext_pgs is an
artifact of the first implementation. And struct mbuf_ext_pgs_data
is a crutch to accomodate the main idea r359919 with minimal churn.

Also, M_EXT of type EXT_PGS are just a synonym of M_NOMAP.

The namespace for the newfeature is somewhat inconsistent and
sometimes has a lengthy prefixes. In these patches we will
gradually bring the namespace to "m_epg" prefix for all mbuf
fields and most functions.

Step 1 of 4:

 o Anonymize mbuf_ext_pgs_data, embed in m_ext
 o Embed mbuf_ext_pgs
 o Start documenting all this entanglement

Reviewed by:	gallatin
Differential Revision:	https://reviews.freebsd.org/D24598

sys/dev/cxgbe/crypto/t4_kern_tls.c

@@ -906,7 +906,7 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
 
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = ntohs(hdr->tls_length);
 
 	/*
@@ -962,7 +962,7 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
 
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = ntohs(hdr->tls_length);
 #ifdef INVARIANTS
 	mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
@@ -1040,7 +1040,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
 		return (wr_len);
 	}
 
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
 	if (tlen < plen) {
 		plen = tlen;
@@ -1064,7 +1064,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
 	wr_len += roundup2(imm_len, 16);
 
 	/* TLS record payload via DSGL. */
-	*nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset,
+	*nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset,
 	    plen - (ext_pgs->hdr_len + offset));
 	wr_len += ktls_sgl_size(*nsegsp);
 
@@ -1543,7 +1543,7 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
 	    (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
 
 	/* Copy the subset of the TLS header requested. */
-	copy_to_txd(&txq->eq, (char *)ext_pgs->m_epg_hdr +
+	copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
 	    mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
 	txq->imm_wrs++;
 
@@ -1604,7 +1604,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 	/* Locate the TLS header. */
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
 
 	/* Determine how much of the TLS record to send. */
@@ -1799,7 +1799,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 
 	/* Recalculate 'nsegs' if cached value is not available. */
 	if (nsegs == 0)
-		nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len +
+		nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len +
 		    offset, plen - (ext_pgs->hdr_len + offset));
 
 	/* Calculate the size of the TLS work request. */
@@ -2031,7 +2031,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 	/* Populate the TLS header */
 	out = (void *)(tx_data + 1);
 	if (offset == 0) {
-		memcpy(out, ext_pgs->m_epg_hdr, ext_pgs->hdr_len);
+		memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
 		out += ext_pgs->hdr_len;
 	}
 
@@ -2067,7 +2067,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 
 	/* SGL for record payload */
 	sglist_reset(txq->gl);
-	if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset,
+	if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset,
 	    plen - (ext_pgs->hdr_len + offset)) != 0) {
 #ifdef INVARIANTS
 		panic("%s: failed to append sglist", __func__);

sys/dev/cxgbe/t4_sge.c

@@ -2435,7 +2435,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
 			off = 0;
 			len -= seglen;
 			paddr = pmap_kextract(
-			    (vm_offset_t)&ext_pgs->m_epg_hdr[segoff]);
+			    (vm_offset_t)&m->m_epg_hdr[segoff]);
 			if (*nextaddr != paddr)
 				nsegs++;
 			*nextaddr = paddr + seglen;
@@ -2454,7 +2454,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
 		off = 0;
 		seglen = min(seglen, len);
 		len -= seglen;
-		paddr = ext_pgs->m_epg_pa[i] + segoff;
+		paddr = m->m_epg_pa[i] + segoff;
 		if (*nextaddr != paddr)
 			nsegs++;
 		*nextaddr = paddr + seglen;
@@ -2463,7 +2463,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
 	if (len != 0) {
 		seglen = min(len, ext_pgs->trail_len - off);
 		len -= seglen;
-		paddr = pmap_kextract((vm_offset_t)&ext_pgs->m_epg_trail[off]);
+		paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
 		if (*nextaddr != paddr)
 			nsegs++;
 		*nextaddr = paddr + seglen;

sys/dev/cxgbe/tom/t4_cpl_io.c

@@ -1935,7 +1935,7 @@ aiotx_free_pgs(struct mbuf *m)
 #endif
 
 	for (int i = 0; i < ext_pgs->npgs; i++) {
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		vm_page_unwire(pg, PQ_ACTIVE);
 	}
 
@@ -2003,7 +2003,7 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
 			    (npages - 2) * PAGE_SIZE;
 		}
 		for (i = 0; i < npages; i++)
-			ext_pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
+			m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
 
 		m->m_len = mlen;
 		m->m_ext.ext_size = npages * PAGE_SIZE;

sys/dev/cxgbe/tom/t4_tls.c

@@ -1623,26 +1623,24 @@ t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
 
 #ifdef KERN_TLS
 static int
-count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs,
-	struct mbuf_ext_pgs_data *ext_pgs_data)
+count_ext_pgs_segs(struct mbuf *m)
 {
 	vm_paddr_t nextpa;
 	u_int i, nsegs;
 
-	MPASS(ext_pgs->npgs > 0);
+	MPASS(m->m_ext_pgs.npgs > 0);
 	nsegs = 1;
-	nextpa = ext_pgs_data->pa[0] + PAGE_SIZE;
-	for (i = 1; i < ext_pgs->npgs; i++) {
-		if (nextpa != ext_pgs_data->pa[i])
+	nextpa = m->m_epg_pa[0] + PAGE_SIZE;
+	for (i = 1; i < m->m_ext_pgs.npgs; i++) {
+		if (nextpa != m->m_epg_pa[i])
 			nsegs++;
-		nextpa = ext_pgs_data->pa[i] + PAGE_SIZE;
+		nextpa = m->m_epg_pa[i] + PAGE_SIZE;
 	}
 	return (nsegs);
 }
 
 static void
-write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
-    struct mbuf_ext_pgs_data *ext_pgs_data, int nsegs)
+write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
 {
 	struct ulptx_sgl *usgl = dst;
 	vm_paddr_t pa;
@@ -1655,37 +1653,37 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
 	    V_ULPTX_NSGE(nsegs));
 
 	/* Figure out the first S/G length. */
-	pa = ext_pgs_data->pa[0] + ext_pgs->first_pg_off;
+	pa = m->m_epg_pa[0] + m->m_ext_pgs.first_pg_off;
 	usgl->addr0 = htobe64(pa);
-	len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
+	len = mbuf_ext_pg_len(&m->m_ext_pgs, 0, m->m_ext_pgs.first_pg_off);
 	pa += len;
-	for (i = 1; i < ext_pgs->npgs; i++) {
-		if (ext_pgs_data->pa[i] != pa)
+	for (i = 1; i < m->m_ext_pgs.npgs; i++) {
+		if (m->m_epg_pa[i] != pa)
 			break;
-		len += mbuf_ext_pg_len(ext_pgs, i, 0);
-		pa += mbuf_ext_pg_len(ext_pgs, i, 0);
+		len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
+		pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 	}
 	usgl->len0 = htobe32(len);
 #ifdef INVARIANTS
 	nsegs--;
 #endif
 
 	j = -1;
-	for (; i < ext_pgs->npgs; i++) {
-		if (j == -1 || ext_pgs_data->pa[i] != pa) {
+	for (; i < m->m_ext_pgs.npgs; i++) {
+		if (j == -1 || m->m_epg_pa[i] != pa) {
 			if (j >= 0)
 				usgl->sge[j / 2].len[j & 1] = htobe32(len);
 			j++;
 #ifdef INVARIANTS
 			nsegs--;
 #endif
-			pa = ext_pgs_data->pa[i];
+			pa = m->m_epg_pa[i];
 			usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
-			len = mbuf_ext_pg_len(ext_pgs, i, 0);
+			len = mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 			pa += len;
 		} else {
-			len += mbuf_ext_pg_len(ext_pgs, i, 0);
-			pa += mbuf_ext_pg_len(ext_pgs, i, 0);
+			len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
+			pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 		}
 	}
 	if (j >= 0) {
@@ -1694,8 +1692,7 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
 		if ((j & 1) == 0)
 			usgl->sge[j / 2].len[1] = htobe32(0);
 	}
-	KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
-	    ext_pgs));
+	KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
 }
 
 /*
@@ -1813,8 +1810,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
 		wr_len += AES_BLOCK_LEN;
 
 		/* Account for SGL in work request length. */
-		nsegs = count_ext_pgs_segs(&m->m_ext_pgs,
-		    &m->m_ext.ext_pgs);
+		nsegs = count_ext_pgs_segs(m);
 		wr_len += sizeof(struct ulptx_sgl) +
 		    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
 
@@ -1892,8 +1888,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
 		memcpy(buf, thdr + 1, toep->tls.iv_len);
 		buf += AES_BLOCK_LEN;
 
-		write_ktlstx_sgl(buf, &m->m_ext_pgs, &m->m_ext.ext_pgs,
-		    nsegs);
+		write_ktlstx_sgl(buf, m, nsegs);
 
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));

sys/kern/kern_mbuf.c

@@ -311,9 +311,6 @@ static void	mb_reclaim(uma_zone_t, int);
 /* Ensure that MSIZE is a power of 2. */
 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
 
-_Static_assert(offsetof(struct mbuf, m_ext) ==
-    offsetof(struct mbuf, m_ext_pgs.m_ext),
-    "m_ext offset mismatch between mbuf and ext_pgs");
 _Static_assert(sizeof(struct mbuf) <= MSIZE,
     "size of mbuf exceeds MSIZE");
 /*
@@ -984,7 +981,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 				goto fail;
 			m_new->m_len = seglen;
 			prev = top = m_new;
-			memcpy(mtod(m_new, void *), &ext_pgs->m_epg_hdr[segoff],
+			memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
 			    seglen);
 		}
 	}
@@ -1002,7 +999,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 		seglen = min(seglen, len);
 		len -= seglen;
 
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		m_new = m_get(M_NOWAIT, MT_DATA);
 		if (m_new == NULL)
 			goto fail;
@@ -1036,7 +1033,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 		else
 			prev->m_next = m_new;
 		m_new->m_len = len;
-		memcpy(mtod(m_new, void *), &ext_pgs->m_epg_trail[off], len);
+		memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
 	}
 
 	if (ref_inc != 0) {
@@ -1154,16 +1151,17 @@ mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
 
 #ifdef INVARIANT_SUPPORT
 void
-mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
+mb_ext_pgs_check(struct mbuf *m)
 {
+	struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
 
 	/*
 	 * NB: This expects a non-empty buffer (npgs > 0 and
 	 * last_pg_len > 0).
 	 */
 	KASSERT(ext_pgs->npgs > 0,
 	    ("ext_pgs with no valid pages: %p", ext_pgs));
-	KASSERT(ext_pgs->npgs <= nitems(ext_pgs->m_epg_pa),
+	KASSERT(ext_pgs->npgs <= nitems(m->m_epg_pa),
 	    ("ext_pgs with too many pages: %p", ext_pgs));
 	KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
 	    ("ext_pgs with too many ready pages: %p", ext_pgs));
@@ -1178,9 +1176,9 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
 		    PAGE_SIZE, ("ext_pgs with single page too large: %p",
 		    ext_pgs));
 	}
-	KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->m_epg_hdr),
+	KASSERT(ext_pgs->hdr_len <= sizeof(m->m_epg_hdr),
 	    ("ext_pgs with too large header length: %p", ext_pgs));
-	KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->m_epg_trail),
+	KASSERT(ext_pgs->trail_len <= sizeof(m->m_epg_trail),
 	    ("ext_pgs with too large header length: %p", ext_pgs));
 }
 #endif

sys/kern/kern_sendfile.c

@@ -203,7 +203,7 @@ sendfile_free_mext_pg(struct mbuf *m)
 	for (i = 0; i < ext_pgs->npgs; i++) {
 		if (cache_last && i == ext_pgs->npgs - 1)
 			flags = 0;
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		vm_page_release(pg, flags);
 	}
 
@@ -1046,11 +1046,11 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
 					ext_pgs->nrdy++;
 				}
 
-				ext_pgs->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
+				m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
 				ext_pgs->npgs++;
 				xfs = xfsize(i, npages, off, space);
 				ext_pgs->last_pg_len = xfs;
-				MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs);
+				MBUF_EXT_PGS_ASSERT_SANITY(m0);
 				mtail->m_len += xfs;
 				mtail->m_ext.ext_size += PAGE_SIZE;
 				continue;

sys/kern/subr_bus_dma.c

@@ -141,7 +141,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
 			off = 0;
 			len -= seglen;
 			error = _bus_dmamap_load_buffer(dmat, map,
-			    &ext_pgs->m_epg_hdr[segoff], seglen, kernel_pmap,
+			    &m->m_epg_hdr[segoff], seglen, kernel_pmap,
 			    flags, segs, nsegs);
 		}
 	}
@@ -159,15 +159,15 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
 		seglen = min(seglen, len);
 		len -= seglen;
 		error = _bus_dmamap_load_phys(dmat, map,
-		    ext_pgs->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
+		    m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
 		pgoff = 0;
 	};
 	if (len != 0 && error == 0) {
 		KASSERT((off + len) <= ext_pgs->trail_len,
 		    ("off + len > trail (%d + %d > %d)", off, len,
 		    ext_pgs->trail_len));
 		error = _bus_dmamap_load_buffer(dmat, map,
-		    &ext_pgs->m_epg_trail[off], len, kernel_pmap, flags, segs,
+		    &m->m_epg_trail[off], len, kernel_pmap, flags, segs,
 		    nsegs);
 	}
 	return (error);