Skip to content

Commit

Permalink
Step 2.5: Stop using 'struct mbuf_ext_pgs' in the kernel itself.
Browse files Browse the repository at this point in the history 
Reviewed by:	gallatin
Differential Revision:	https://reviews.freebsd.org/D24598
  • Loading branch information
@glebius
glebius committed May 3, 2020
1 parent b363a43 commit bccf6e2
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 102 deletions.
40 changes: 18 additions & 22 deletions sys/kern/kern_mbuf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -934,7 +934,6 @@ mb_unmapped_free_mext(struct mbuf *m)
static struct mbuf *
_mb_unmapped_to_ext(struct mbuf *m)
{
struct mbuf_ext_pgs *ext_pgs;
struct mbuf *m_new, *top, *prev, *mref;
struct sf_buf *sf;
vm_page_t pg;
Expand All @@ -943,9 +942,8 @@ _mb_unmapped_to_ext(struct mbuf *m)
u_int ref_inc = 0;

MBUF_EXT_PGS_ASSERT(m);
ext_pgs = &m->m_ext_pgs;
len = m->m_len;
KASSERT(ext_pgs->tls == NULL, ("%s: can't convert TLS mbuf %p",
KASSERT(m->m_ext_pgs.tls == NULL, ("%s: can't convert TLS mbuf %p",
__func__, m));

/* See if this is the mbuf that holds the embedded refcount. */
Expand All @@ -963,11 +961,11 @@ _mb_unmapped_to_ext(struct mbuf *m)
off = mtod(m, vm_offset_t);

top = NULL;
if (ext_pgs->hdr_len != 0) {
if (off >= ext_pgs->hdr_len) {
off -= ext_pgs->hdr_len;
if (m->m_ext_pgs.hdr_len != 0) {
if (off >= m->m_ext_pgs.hdr_len) {
off -= m->m_ext_pgs.hdr_len;
} else {
seglen = ext_pgs->hdr_len - off;
seglen = m->m_ext_pgs.hdr_len - off;
segoff = off;
seglen = min(seglen, len);
off = 0;
Expand All @@ -981,8 +979,8 @@ _mb_unmapped_to_ext(struct mbuf *m)
seglen);
}
}
pgoff = ext_pgs->first_pg_off;
for (i = 0; i < ext_pgs->npgs && len > 0; i++) {
pgoff = m->m_ext_pgs.first_pg_off;
for (i = 0; i < m->m_ext_pgs.npgs && len > 0; i++) {
pglen = m_epg_pagelen(m, i, pgoff);
if (off >= pglen) {
off -= pglen;
Expand Down Expand Up @@ -1018,9 +1016,9 @@ _mb_unmapped_to_ext(struct mbuf *m)
pgoff = 0;
};
if (len != 0) {
KASSERT((off + len) <= ext_pgs->trail_len,
KASSERT((off + len) <= m->m_ext_pgs.trail_len,
("off + len > trail (%d + %d > %d)", off, len,
ext_pgs->trail_len));
m->m_ext_pgs.trail_len));
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
Expand Down Expand Up @@ -1119,22 +1117,20 @@ struct mbuf *
mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
{
struct mbuf *m;
struct mbuf_ext_pgs *ext_pgs;

m = m_get(how, MT_DATA);
if (m == NULL)
return (NULL);

ext_pgs = &m->m_ext_pgs;
ext_pgs->npgs = 0;
ext_pgs->nrdy = 0;
ext_pgs->first_pg_off = 0;
ext_pgs->last_pg_len = 0;
ext_pgs->flags = 0;
ext_pgs->hdr_len = 0;
ext_pgs->trail_len = 0;
ext_pgs->tls = NULL;
ext_pgs->so = NULL;
m->m_ext_pgs.npgs = 0;
m->m_ext_pgs.nrdy = 0;
m->m_ext_pgs.first_pg_off = 0;
m->m_ext_pgs.last_pg_len = 0;
m->m_ext_pgs.flags = 0;
m->m_ext_pgs.hdr_len = 0;
m->m_ext_pgs.trail_len = 0;
m->m_ext_pgs.tls = NULL;
m->m_ext_pgs.so = NULL;
m->m_data = NULL;
m->m_flags |= (M_EXT | M_RDONLY | M_NOMAP);
m->m_ext.ext_type = EXT_PGS;
Expand Down
16 changes: 6 additions & 10 deletions sys/kern/kern_sendfile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,6 @@ sendfile_free_mext(struct mbuf *m)
static void
sendfile_free_mext_pg(struct mbuf *m)
{
struct mbuf_ext_pgs *ext_pgs;
vm_page_t pg;
int flags, i;
bool cache_last;
Expand All @@ -197,11 +196,10 @@ sendfile_free_mext_pg(struct mbuf *m)
("%s: m %p !M_EXT or !EXT_PGS", __func__, m));

cache_last = m->m_ext.ext_flags & EXT_FLAG_CACHE_LAST;
ext_pgs = &m->m_ext_pgs;
flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;

for (i = 0; i < ext_pgs->npgs; i++) {
if (cache_last && i == ext_pgs->npgs - 1)
for (i = 0; i < m->m_ext_pgs.npgs; i++) {
if (cache_last && i == m->m_ext_pgs.npgs - 1)
flags = 0;
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_release(pg, flags);
Expand Down Expand Up @@ -692,7 +690,6 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
#ifdef KERN_TLS
struct ktls_session *tls;
#endif
struct mbuf_ext_pgs *ext_pgs;
struct mbuf *m, *mh, *mhtail;
struct sf_buf *sf;
struct shmfd *shmfd;
Expand Down Expand Up @@ -1029,7 +1026,6 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
sfs->count++;
mtx_unlock(&sfs->mtx);
}
ext_pgs = &m0->m_ext_pgs;
ext_pgs_idx = 0;

/* Append to mbuf chain. */
Expand All @@ -1038,18 +1034,18 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
else
m = m0;
mtail = m0;
ext_pgs->first_pg_off =
m0->m_ext_pgs.first_pg_off =
vmoff(i, off) & PAGE_MASK;
}
if (nios) {
mtail->m_flags |= M_NOTREADY;
ext_pgs->nrdy++;
m0->m_ext_pgs.nrdy++;
}

m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
ext_pgs->npgs++;
m0->m_ext_pgs.npgs++;
xfs = xfsize(i, npages, off, space);
ext_pgs->last_pg_len = xfs;
m0->m_ext_pgs.last_pg_len = xfs;
MBUF_EXT_PGS_ASSERT_SANITY(m0);
mtail->m_len += xfs;
mtail->m_ext.ext_size += PAGE_SIZE;
Expand Down
65 changes: 26 additions & 39 deletions sys/kern/uipc_ktls.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1287,14 +1287,12 @@ ktls_destroy(struct ktls_session *tls)
void
ktls_seq(struct sockbuf *sb, struct mbuf *m)
{
struct mbuf_ext_pgs *pgs;

for (; m != NULL; m = m->m_next) {
KASSERT((m->m_flags & M_NOMAP) != 0,
("ktls_seq: mapped mbuf %p", m));

pgs = &m->m_ext_pgs;
pgs->seqno = sb->sb_tls_seqno;
m->m_ext_pgs.seqno = sb->sb_tls_seqno;
sb->sb_tls_seqno++;
}
}
Expand All @@ -1318,7 +1316,6 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
{
struct tls_record_layer *tlshdr;
struct mbuf *m;
struct mbuf_ext_pgs *pgs;
uint64_t *noncep;
uint16_t tls_len;
int maxlen;
Expand All @@ -1341,13 +1338,12 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
("ktls_frame: mapped mbuf %p (top = %p)\n", m, top));

tls_len = m->m_len;
pgs = &m->m_ext_pgs;

/* Save a reference to the session. */
pgs->tls = ktls_hold(tls);
m->m_ext_pgs.tls = ktls_hold(tls);

pgs->hdr_len = tls->params.tls_hlen;
pgs->trail_len = tls->params.tls_tlen;
m->m_ext_pgs.hdr_len = tls->params.tls_hlen;
m->m_ext_pgs.trail_len = tls->params.tls_tlen;
if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {
int bs, delta;

Expand All @@ -1369,9 +1365,9 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
*/
bs = tls->params.tls_bs;
delta = (tls_len + tls->params.tls_tlen) & (bs - 1);
pgs->trail_len -= delta;
m->m_ext_pgs.trail_len -= delta;
}
m->m_len += pgs->hdr_len + pgs->trail_len;
m->m_len += m->m_ext_pgs.hdr_len + m->m_ext_pgs.trail_len;

/* Populate the TLS header. */
tlshdr = (void *)m->m_epg_hdr;
Expand All @@ -1386,7 +1382,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
tlshdr->tls_vminor = TLS_MINOR_VER_TWO;
tlshdr->tls_type = TLS_RLTYPE_APP;
/* save the real record type for later */
pgs->record_type = record_type;
m->m_ext_pgs.record_type = record_type;
m->m_epg_trail[0] = record_type;
} else {
tlshdr->tls_vminor = tls->params.tls_vminor;
Expand Down Expand Up @@ -1423,24 +1419,21 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
*/
if (tls->mode == TCP_TLS_MODE_SW) {
m->m_flags |= M_NOTREADY;
pgs->nrdy = pgs->npgs;
*enq_cnt += pgs->npgs;
m->m_ext_pgs.nrdy = m->m_ext_pgs.npgs;
*enq_cnt += m->m_ext_pgs.npgs;
}
}
}

void
ktls_enqueue_to_free(struct mbuf *m)
{
struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;

pgs = &m->m_ext_pgs;

/* Mark it for freeing. */
pgs->flags |= EPG_FLAG_2FREE;
wq = &ktls_wq[pgs->tls->wq_index];
m->m_ext_pgs.flags |= EPG_FLAG_2FREE;
wq = &ktls_wq[m->m_ext_pgs.tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
Expand All @@ -1452,7 +1445,6 @@ ktls_enqueue_to_free(struct mbuf *m)
void
ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
{
struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;

Expand All @@ -1461,19 +1453,17 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
("ktls_enqueue: %p not unready & nomap mbuf\n", m));
KASSERT(page_count != 0, ("enqueueing TLS mbuf with zero page count"));

pgs = &m->m_ext_pgs;

KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
KASSERT(m->m_ext_pgs.tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));

pgs->enc_cnt = page_count;
m->m_ext_pgs.enc_cnt = page_count;

/*
* Save a pointer to the socket. The caller is responsible
* for taking an additional reference via soref().
*/
pgs->so = so;
m->m_ext_pgs.so = so;

wq = &ktls_wq[pgs->tls->wq_index];
wq = &ktls_wq[m->m_ext_pgs.tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
Expand All @@ -1489,7 +1479,6 @@ ktls_encrypt(struct mbuf *top)
struct ktls_session *tls;
struct socket *so;
struct mbuf *m;
struct mbuf_ext_pgs *pgs;
vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
Expand Down Expand Up @@ -1526,15 +1515,13 @@ ktls_encrypt(struct mbuf *top)
*/
error = 0;
for (m = top; npages != total_pages; m = m->m_next) {
pgs = &m->m_ext_pgs;

KASSERT(pgs->tls == tls,
KASSERT(m->m_ext_pgs.tls == tls,
("different TLS sessions in a single mbuf chain: %p vs %p",
tls, pgs->tls));
tls, m->m_ext_pgs.tls));
KASSERT((m->m_flags & (M_NOMAP | M_NOTREADY)) ==
(M_NOMAP | M_NOTREADY),
("%p not unready & nomap mbuf (top = %p)\n", m, top));
KASSERT(npages + pgs->npgs <= total_pages,
KASSERT(npages + m->m_ext_pgs.npgs <= total_pages,
("page count mismatch: top %p, total_pages %d, m %p", top,
total_pages, m));

Expand All @@ -1546,10 +1533,10 @@ ktls_encrypt(struct mbuf *top)
* (from sendfile), anonymous wired pages are
* allocated and assigned to the destination iovec.
*/
is_anon = (pgs->flags & EPG_FLAG_ANON) != 0;
is_anon = (m->m_ext_pgs.flags & EPG_FLAG_ANON) != 0;

off = pgs->first_pg_off;
for (i = 0; i < pgs->npgs; i++, off = 0) {
off = m->m_ext_pgs.first_pg_off;
for (i = 0; i < m->m_ext_pgs.npgs; i++, off = 0) {
len = m_epg_pagelen(m, i, off);
src_iov[i].iov_len = len;
src_iov[i].iov_base =
Expand Down Expand Up @@ -1578,8 +1565,8 @@ ktls_encrypt(struct mbuf *top)

error = (*tls->sw_encrypt)(tls,
(const struct tls_record_layer *)m->m_epg_hdr,
m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
pgs->record_type);
m->m_epg_trail, src_iov, dst_iov, i, m->m_ext_pgs.seqno,
m->m_ext_pgs.record_type);
if (error) {
counter_u64_add(ktls_offload_failed_crypto, 1);
break;
Expand All @@ -1595,14 +1582,14 @@ ktls_encrypt(struct mbuf *top)
m->m_ext.ext_free(m);

/* Replace them with the new pages. */
for (i = 0; i < pgs->npgs; i++)
for (i = 0; i < m->m_ext_pgs.npgs; i++)
m->m_epg_pa[i] = parray[i];

/* Use the basic free routine. */
m->m_ext.ext_free = mb_free_mext_pgs;

/* Pages are now writable. */
pgs->flags |= EPG_FLAG_ANON;
m->m_ext_pgs.flags |= EPG_FLAG_ANON;
}

/*
Expand All @@ -1612,7 +1599,7 @@ ktls_encrypt(struct mbuf *top)
* yet-to-be-encrypted records having an associated
* session.
*/
pgs->tls = NULL;
m->m_ext_pgs.tls = NULL;
ktls_free(tls);
}

Expand Down
Loading

0 comments on commit bccf6e2

Please sign in to comment.