Skip to content

Commit

Permalink
aio: micro-optimize the lio_opcode assignments
Browse files Browse the repository at this point in the history 
This allows slightly more efficient opcode testing in-kernel.  It is
transparent to userland, except to applications that sneakily submit
aio fsync or aio mlock operations via lio_listio, which has never been
documented, requires the use of deliberately undefined constants
(LIO_SYNC and LIO_MLOCK), and is arguably a bug.

Reviewed by:	jhb
Differential Revision:	https://reviews.freebsd.org/D27942
  • Loading branch information
@asomers
asomers committed Jan 20, 2021
1 parent b84d0aa commit 2247f48
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 44 deletions.
8 changes: 3 additions & 5 deletions sys/kern/sys_socket.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -775,10 +775,10 @@ soo_aio_cancel(struct kaiocb *job)

so = job->fd_file->f_data;
opcode = job->uaiocb.aio_lio_opcode;
if (opcode == LIO_READ || opcode == LIO_READV)
if (opcode & LIO_READ)
sb = &so->so_rcv;
else {
MPASS(opcode == LIO_WRITE || opcode == LIO_WRITEV);
MPASS(opcode & LIO_WRITE);
sb = &so->so_snd;
}

Expand Down Expand Up @@ -808,13 +808,11 @@ soo_aio_queue(struct file *fp, struct kaiocb *job)
if (error == 0)
return (0);

switch (job->uaiocb.aio_lio_opcode) {
switch (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) {
case LIO_READ:
case LIO_READV:
sb = &so->so_rcv;
break;
case LIO_WRITE:
case LIO_WRITEV:
sb = &so->so_snd;
break;
default:
Expand Down
48 changes: 14 additions & 34 deletions sys/kern/vfs_aio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -817,8 +817,7 @@ aio_process_rw(struct kaiocb *job)
if (error != 0 && job->uiop->uio_resid != cnt) {
if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
error = 0;
if (error == EPIPE &&
(opcode == LIO_WRITE || opcode == LIO_WRITEV)) {
if (error == EPIPE && (opcode & LIO_WRITE)) {
PROC_LOCK(job->userproc);
kern_psignal(job->userproc, SIGPIPE);
PROC_UNLOCK(job->userproc);
Expand All @@ -841,8 +840,7 @@ aio_process_sync(struct kaiocb *job)
struct file *fp = job->fd_file;
int error = 0;

KASSERT(job->uaiocb.aio_lio_opcode == LIO_SYNC ||
job->uaiocb.aio_lio_opcode == LIO_DSYNC,
KASSERT(job->uaiocb.aio_lio_opcode & LIO_SYNC,
("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode));

td->td_ucred = job->cred;
Expand Down Expand Up @@ -1239,8 +1237,7 @@ aio_qbio(struct proc *p, struct kaiocb *job)
if (vp->v_bufobj.bo_bsize == 0)
return (-1);

bio_cmd = opcode == LIO_WRITE || opcode == LIO_WRITEV ? BIO_WRITE :
BIO_READ;
bio_cmd = (opcode & LIO_WRITE) ? BIO_WRITE : BIO_READ;
iovcnt = job->uiop->uio_iovcnt;
if (iovcnt > max_buf_aio)
return (-1);
Expand Down Expand Up @@ -1422,7 +1419,7 @@ aiocb_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
error = copyin(ujob, kcb, sizeof(struct aiocb));
if (error)
return (error);
if (type == LIO_READV || type == LIO_WRITEV) {
if (type & LIO_VECTORED) {
/* malloc a uio and copy in the iovec */
error = copyinuio(__DEVOLATILE(struct iovec*, kcb->aio_iov),
kcb->aio_iovcnt, &kjob->uiop);
Expand Down Expand Up @@ -1609,7 +1606,7 @@ aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj,
if (error)
goto err3;

if ((opcode == LIO_SYNC || opcode == LIO_DSYNC) && fp->f_vnode == NULL) {
if ((opcode & LIO_SYNC) && fp->f_vnode == NULL) {
error = EINVAL;
goto err3;
}
Expand Down Expand Up @@ -1669,33 +1666,24 @@ aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj,
job->jobflags = KAIOCB_QUEUEING;
job->lio = lj;

switch (opcode) {
case LIO_READV:
case LIO_WRITEV:
if (opcode & LIO_VECTORED) {
/* Use the uio copied in by aio_copyin */
MPASS(job->uiop != &job->uio && job->uiop != NULL);
break;
case LIO_READ:
case LIO_WRITE:
} else {
/* Setup the inline uio */
job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf;
job->iov[0].iov_len = job->uaiocb.aio_nbytes;
job->uio.uio_iov = job->iov;
job->uio.uio_iovcnt = 1;
job->uio.uio_resid = job->uaiocb.aio_nbytes;
job->uio.uio_segflg = UIO_USERSPACE;
/* FALLTHROUGH */
default:
job->uiop = &job->uio;
break;
}
switch (opcode) {
switch (opcode & (LIO_READ | LIO_WRITE)) {
case LIO_READ:
case LIO_READV:
job->uiop->uio_rw = UIO_READ;
break;
case LIO_WRITE:
case LIO_WRITEV:
job->uiop->uio_rw = UIO_WRITE;
break;
}
Expand Down Expand Up @@ -1813,21 +1801,14 @@ aio_queue_file(struct file *fp, struct kaiocb *job)
return (EOPNOTSUPP);
}

switch (job->uaiocb.aio_lio_opcode) {
case LIO_READ:
case LIO_READV:
case LIO_WRITE:
case LIO_WRITEV:
if (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) {
aio_schedule(job, aio_process_rw);
error = 0;
break;
case LIO_SYNC:
case LIO_DSYNC:
} else if (job->uaiocb.aio_lio_opcode & LIO_SYNC) {
AIO_LOCK(ki);
TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) {
if (job2->fd_file == job->fd_file &&
job2->uaiocb.aio_lio_opcode != LIO_SYNC &&
job2->uaiocb.aio_lio_opcode != LIO_DSYNC &&
((job2->uaiocb.aio_lio_opcode & LIO_SYNC) == 0) &&
job2->seqno < job->seqno) {
job2->jobflags |= KAIOCB_CHECKSYNC;
job->pending++;
Expand All @@ -1847,8 +1828,7 @@ aio_queue_file(struct file *fp, struct kaiocb *job)
AIO_UNLOCK(ki);
aio_schedule(job, aio_process_sync);
error = 0;
break;
default:
} else {
error = EINVAL;
}
return (error);
Expand Down Expand Up @@ -2498,7 +2478,7 @@ aio_biowakeup(struct bio *bp)
*/
if (flags & BIO_ERROR)
atomic_set_int(&job->error, bio_error);
if (opcode == LIO_WRITE || opcode == LIO_WRITEV)
if (opcode & LIO_WRITE)
atomic_add_int(&job->outblock, nblks);
else
atomic_add_int(&job->inblock, nblks);
Expand Down Expand Up @@ -2836,7 +2816,7 @@ aiocb32_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
CP(job32, *kcb, aio_fildes);
CP(job32, *kcb, aio_offset);
CP(job32, *kcb, aio_lio_opcode);
if (type == LIO_READV || type == LIO_WRITEV) {
if (type & LIO_VECTORED) {
iov32 = PTRIN(job32.aio_iov);
CP(job32, *kcb, aio_iovcnt);
/* malloc a uio and copy in the iovec */
Expand Down
11 changes: 6 additions & 5 deletions sys/sys/aio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,12 @@
#define LIO_WRITE 0x1
#define LIO_READ 0x2
#if defined(_KERNEL) || defined(_WANT_ALL_LIO_OPCODES)
#define LIO_SYNC 0x3
#define LIO_MLOCK 0x4
#define LIO_WRITEV 0x5
#define LIO_READV 0x6
#define LIO_DSYNC 0x7
#define LIO_VECTORED 0x4
#define LIO_WRITEV (LIO_WRITE | LIO_VECTORED)
#define LIO_READV (LIO_READ | LIO_VECTORED)
#define LIO_SYNC 0x8
#define LIO_DSYNC (0x10 | LIO_SYNC)
#define LIO_MLOCK 0x20
#endif

/*
Expand Down

0 comments on commit 2247f48

Please sign in to comment.