Skip to content

Commit

Permalink
[PATCH] uml: TLB operation batching
Browse files Browse the repository at this point in the history 
This adds VM op batching to skas0.  Rather than having a context switch to and
from the userspace stub for each address space change, we write a number of
operations to the stub data page and invoke a different stub which loops over
them and executes them all in one go.

The operations are stored as [ system call number, arg1, arg2, ... ] tuples.

The set is terminated by a system call number of 0.  Single operations, i.e.
page faults, are handled in the old way, since that is slightly more
efficient.

For a kernel build, a minority (~1/4) of the operations are part of a set.
These sets averaged ~100 in length, so for this quarter, the context switching
overhead is greatly reduced.

Signed-off-by: Jeff Dike <[email protected]>
Cc: Paolo Giarrusso <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@cfd-36
cfd-36 authored and Linus Torvalds committed Sep 5, 2005
1 parent 77fa5ad commit c560049
Show file tree
Hide file tree
Showing 8 changed files with 273 additions and 166 deletions.
22 changes: 4 additions & 18 deletions arch/um/include/tlb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
#include "um_mmu.h"

struct host_vm_op {
enum { MMAP, MUNMAP, MPROTECT } type;
enum { NONE, MMAP, MUNMAP, MPROTECT } type;
union {
struct {
unsigned long addr;
Expand Down Expand Up @@ -38,24 +38,10 @@ extern void mprotect_kernel_vm(int w);
extern void force_flush_all(void);
extern void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
unsigned long end_addr, int force,
void (*do_ops)(union mm_context *,
struct host_vm_op *, int));
void *(*do_ops)(union mm_context *,
struct host_vm_op *, int, int,
void *));
extern int flush_tlb_kernel_range_common(unsigned long start,
unsigned long end);

extern int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
int r, int w, int x, struct host_vm_op *ops, int index,
int last_filled, union mm_context *mmu,
void (*do_ops)(union mm_context *, struct host_vm_op *,
int));
extern int add_munmap(unsigned long addr, unsigned long len,
struct host_vm_op *ops, int index, int last_filled,
union mm_context *mmu,
void (*do_ops)(union mm_context *, struct host_vm_op *,
int));
extern int add_mprotect(unsigned long addr, unsigned long len, int r, int w,
int x, struct host_vm_op *ops, int index,
int last_filled, union mm_context *mmu,
void (*do_ops)(union mm_context *, struct host_vm_op *,
int));
#endif
13 changes: 8 additions & 5 deletions arch/um/kernel/skas/include/skas.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@ extern void new_thread_proc(void *stack, void (*handler)(int sig));
extern void remove_sigstack(void);
extern void new_thread_handler(int sig);
extern void handle_syscall(union uml_pt_regs *regs);
extern int map(struct mm_id * mm_idp, unsigned long virt, unsigned long len,
int r, int w, int x, int phys_fd, unsigned long long offset);
extern int unmap(struct mm_id * mm_idp, void *addr, unsigned long len);
extern int protect(struct mm_id * mm_idp, unsigned long addr,
unsigned long len, int r, int w, int x);
extern void *map(struct mm_id * mm_idp, unsigned long virt,
unsigned long len, int r, int w, int x, int phys_fd,
unsigned long long offset, int done, void *data);
extern void *unmap(struct mm_id * mm_idp, void *addr,
unsigned long len, int done, void *data);
extern void *protect(struct mm_id * mm_idp, unsigned long addr,
unsigned long len, int r, int w, int x, int done,
void *data);
extern void user_signal(int sig, union uml_pt_regs *regs, int pid);
extern int new_mm(int from);
extern int start_userspace(unsigned long stub_stack);
Expand Down
112 changes: 89 additions & 23 deletions arch/um/kernel/skas/mem_user.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@
#include "sysdep/stub.h"
#include "skas.h"

extern unsigned long syscall_stub, __syscall_stub_start;
extern unsigned long syscall_stub, batch_syscall_stub, __syscall_stub_start;

extern void wait_stub_done(int pid, int sig, char * fname);

static long run_syscall_stub(struct mm_id * mm_idp, int syscall,
unsigned long *args)
int single_count = 0;

static long one_syscall_stub(struct mm_id * mm_idp, int syscall,
unsigned long *args)
{
int n, pid = mm_idp->u.pid;
unsigned long regs[MAX_REG_NR];
Expand All @@ -49,18 +51,80 @@ static long run_syscall_stub(struct mm_id * mm_idp, int syscall,
regs[REGS_SYSCALL_ARG6] = args[5];
n = ptrace_setregs(pid, regs);
if(n < 0){
printk("run_syscall_stub : PTRACE_SETREGS failed, "
printk("one_syscall_stub : PTRACE_SETREGS failed, "
"errno = %d\n", n);
return(n);
}

wait_stub_done(pid, 0, "one_syscall_stub");

return(*((unsigned long *) mm_idp->stack));
}

int multi_count = 0;
int multi_op_count = 0;

static long many_syscall_stub(struct mm_id * mm_idp, int syscall,
unsigned long *args, int done, void **addr_out)
{
unsigned long regs[MAX_REG_NR], *stack;
int n, pid = mm_idp->u.pid;

stack = *addr_out;
if(stack == NULL)
stack = (unsigned long *) current_stub_stack();
*stack++ = syscall;
*stack++ = args[0];
*stack++ = args[1];
*stack++ = args[2];
*stack++ = args[3];
*stack++ = args[4];
*stack++ = args[5];
*stack = 0;
multi_op_count++;

if(!done && ((((unsigned long) stack) & ~PAGE_MASK) <
PAGE_SIZE - 8 * sizeof(long))){
*addr_out = stack;
return 0;
}

multi_count++;
get_safe_registers(regs);
regs[REGS_IP_INDEX] = UML_CONFIG_STUB_CODE +
((unsigned long) &batch_syscall_stub -
(unsigned long) &__syscall_stub_start);
regs[REGS_SP_INDEX] = UML_CONFIG_STUB_DATA;

n = ptrace_setregs(pid, regs);
if(n < 0){
printk("many_syscall_stub : PTRACE_SETREGS failed, "
"errno = %d\n", n);
return(n);
}

wait_stub_done(pid, 0, "run_syscall_stub");
wait_stub_done(pid, 0, "many_syscall_stub");
stack = (unsigned long *) mm_idp->stack;

return(*((unsigned long *) mm_idp->stack));
*addr_out = stack;
return(*stack);
}

int map(struct mm_id *mm_idp, unsigned long virt, unsigned long len,
int r, int w, int x, int phys_fd, unsigned long long offset)
static long run_syscall_stub(struct mm_id * mm_idp, int syscall,
unsigned long *args, void **addr, int done)
{
long res;

if((*addr == NULL) && done)
res = one_syscall_stub(mm_idp, syscall, args);
else res = many_syscall_stub(mm_idp, syscall, args, done, addr);

return res;
}

void *map(struct mm_id * mm_idp, unsigned long virt, unsigned long len,
int r, int w, int x, int phys_fd, unsigned long long offset,
int done, void *data)
{
int prot, n;

Expand All @@ -70,6 +134,7 @@ int map(struct mm_id *mm_idp, unsigned long virt, unsigned long len,
if(proc_mm){
struct proc_mm_op map;
int fd = mm_idp->u.mm_fd;

map = ((struct proc_mm_op) { .op = MM_MMAP,
.u =
{ .mmap =
Expand All @@ -91,50 +156,50 @@ int map(struct mm_id *mm_idp, unsigned long virt, unsigned long len,
MAP_SHARED | MAP_FIXED, phys_fd,
MMAP_OFFSET(offset) };

res = run_syscall_stub(mm_idp, STUB_MMAP_NR, args);
res = run_syscall_stub(mm_idp, STUB_MMAP_NR, args,
&data, done);
if((void *) res == MAP_FAILED)
printk("mmap stub failed, errno = %d\n", res);
}

return 0;
return data;
}

int unmap(struct mm_id *mm_idp, void *addr, unsigned long len)
void *unmap(struct mm_id * mm_idp, void *addr, unsigned long len, int done,
void *data)
{
int n;

if(proc_mm){
struct proc_mm_op unmap;
int fd = mm_idp->u.mm_fd;

unmap = ((struct proc_mm_op) { .op = MM_MUNMAP,
.u =
{ .munmap =
{ .addr =
(unsigned long) addr,
.len = len } } } );
n = os_write_file(fd, &unmap, sizeof(unmap));
if(n != sizeof(unmap)) {
if(n < 0)
return(n);
else if(n > 0)
return(-EIO);
}
if(n != sizeof(unmap))
printk("unmap - proc_mm write returned %d\n", n);
}
else {
int res;
unsigned long args[] = { (unsigned long) addr, len, 0, 0, 0,
0 };

res = run_syscall_stub(mm_idp, __NR_munmap, args);
res = run_syscall_stub(mm_idp, __NR_munmap, args,
&data, done);
if(res < 0)
printk("munmap stub failed, errno = %d\n", res);
}

return(0);
return data;
}

int protect(struct mm_id *mm_idp, unsigned long addr, unsigned long len,
int r, int w, int x)
void *protect(struct mm_id * mm_idp, unsigned long addr, unsigned long len,
int r, int w, int x, int done, void *data)
{
struct proc_mm_op protect;
int prot, n;
Expand All @@ -160,12 +225,13 @@ int protect(struct mm_id *mm_idp, unsigned long addr, unsigned long len,
int res;
unsigned long args[] = { addr, len, prot, 0, 0, 0 };

res = run_syscall_stub(mm_idp, __NR_mprotect, args);
res = run_syscall_stub(mm_idp, __NR_mprotect, args,
&data, done);
if(res < 0)
panic("mprotect stub failed, errno = %d\n", res);
}

return(0);
return data;
}

void before_mem_skas(unsigned long unused)
Expand Down
23 changes: 14 additions & 9 deletions arch/um/kernel/skas/tlb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
#include "os.h"
#include "tlb.h"

static void do_ops(union mm_context *mmu, struct host_vm_op *ops, int last)
static void *do_ops(union mm_context *mmu, struct host_vm_op *ops, int last,
int finished, void *flush)
{
struct host_vm_op *op;
int i;
Expand All @@ -27,24 +28,28 @@ static void do_ops(union mm_context *mmu, struct host_vm_op *ops, int last)
op = &ops[i];
switch(op->type){
case MMAP:
map(&mmu->skas.id, op->u.mmap.addr, op->u.mmap.len,
op->u.mmap.r, op->u.mmap.w, op->u.mmap.x,
op->u.mmap.fd, op->u.mmap.offset);
flush = map(&mmu->skas.id, op->u.mmap.addr,
op->u.mmap.len, op->u.mmap.r, op->u.mmap.w,
op->u.mmap.x, op->u.mmap.fd,
op->u.mmap.offset, finished, flush);
break;
case MUNMAP:
unmap(&mmu->skas.id, (void *) op->u.munmap.addr,
op->u.munmap.len);
flush = unmap(&mmu->skas.id, (void *) op->u.munmap.addr,
op->u.munmap.len, finished, flush);
break;
case MPROTECT:
protect(&mmu->skas.id, op->u.mprotect.addr,
op->u.mprotect.len, op->u.mprotect.r,
op->u.mprotect.w, op->u.mprotect.x);
flush = protect(&mmu->skas.id, op->u.mprotect.addr,
op->u.mprotect.len, op->u.mprotect.r,
op->u.mprotect.w, op->u.mprotect.x,
finished, flush);
break;
default:
printk("Unknown op type %d in do_ops\n", op->type);
break;
}
}

return flush;
}

extern int proc_mm;
Expand Down
Loading

0 comments on commit c560049

Please sign in to comment.