Skip to content

Commit

Permalink
fs/binfmt_elf.c: better codegen around current->mm
Browse files Browse the repository at this point in the history 
"current->mm" pointer is stable in general except few cases one of which
execve(2).  Compiler can't treat is as stable but it _is_ stable most of
the time.  During ELF loading process ->mm becomes stable right after
flush_old_exec().

Help compiler by caching current->mm, otherwise it continues to refetch
it.

	add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-141 (-141)
	Function                                     old     new   delta
	elf_core_dump                               5062    5039     -23
	load_elf_binary                             5426    5308    -118

Note: other cases are left as is because it is either pessimisation or
no change in binary size.

Link: http://lkml.kernel.org/r/20191215124755.GB21124@avx2
Signed-off-by: Alexey Dobriyan <[email protected]>
Reviewed-by: Andrew Morton <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@torvalds
Alexey Dobriyan authored and torvalds committed Jan 31, 2020
1 parent a62c5b1 commit 03c6d72
Showing 1 changed file with 28 additions and 24 deletions.
52 changes: 28 additions & 24 deletions fs/binfmt_elf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
unsigned long load_addr, unsigned long interp_load_addr,
unsigned long e_entry)
{
struct mm_struct *mm = current->mm;
unsigned long p = bprm->p;
int argc = bprm->argc;
int envc = bprm->envc;
Expand Down Expand Up @@ -227,7 +228,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
return -EFAULT;

/* Create the ELF interpreter info */
elf_info = (elf_addr_t *)current->mm->saved_auxv;
elf_info = (elf_addr_t *)mm->saved_auxv;
/* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */
#define NEW_AUX_ENT(id, val) \
do { \
Expand Down Expand Up @@ -276,13 +277,13 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
}
#undef NEW_AUX_ENT
/* AT_NULL is zero; clear the rest too */
memset(elf_info, 0, (char *)current->mm->saved_auxv +
sizeof(current->mm->saved_auxv) - (char *)elf_info);
memset(elf_info, 0, (char *)mm->saved_auxv +
sizeof(mm->saved_auxv) - (char *)elf_info);

/* And advance past the AT_NULL entry. */
elf_info += 2;

ei_index = elf_info - (elf_addr_t *)current->mm->saved_auxv;
ei_index = elf_info - (elf_addr_t *)mm->saved_auxv;
sp = STACK_ADD(p, ei_index);

items = (argc + 1) + (envc + 1) + 1;
Expand All @@ -301,7 +302,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
* Grow the stack manually; some architectures have a limit on how
* far ahead a user-space access may be in order to grow the stack.
*/
vma = find_extend_vma(current->mm, bprm->p);
vma = find_extend_vma(mm, bprm->p);
if (!vma)
return -EFAULT;

Expand All @@ -310,7 +311,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
return -EFAULT;

/* Populate list of argv pointers back to argv strings. */
p = current->mm->arg_end = current->mm->arg_start;
p = mm->arg_end = mm->arg_start;
while (argc-- > 0) {
size_t len;
if (__put_user((elf_addr_t)p, sp++))
Expand All @@ -322,10 +323,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
}
if (__put_user(0, sp++))
return -EFAULT;
current->mm->arg_end = p;
mm->arg_end = p;

/* Populate list of envp pointers back to envp strings. */
current->mm->env_end = current->mm->env_start = p;
mm->env_end = mm->env_start = p;
while (envc-- > 0) {
size_t len;
if (__put_user((elf_addr_t)p, sp++))
Expand All @@ -337,10 +338,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
}
if (__put_user(0, sp++))
return -EFAULT;
current->mm->env_end = p;
mm->env_end = p;

/* Put the elf_info on the stack in the right place. */
if (copy_to_user(sp, current->mm->saved_auxv, ei_index * sizeof(elf_addr_t)))
if (copy_to_user(sp, mm->saved_auxv, ei_index * sizeof(elf_addr_t)))
return -EFAULT;
return 0;
}
Expand Down Expand Up @@ -701,6 +702,7 @@ static int load_elf_binary(struct linux_binprm *bprm)
struct elfhdr interp_elf_ex;
} *loc;
struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE;
struct mm_struct *mm;
struct pt_regs *regs;

loc = kmalloc(sizeof(*loc), GFP_KERNEL);
Expand Down Expand Up @@ -1096,11 +1098,13 @@ static int load_elf_binary(struct linux_binprm *bprm)
load_addr, interp_load_addr, e_entry);
if (retval < 0)
goto out;
current->mm->end_code = end_code;
current->mm->start_code = start_code;
current->mm->start_data = start_data;
current->mm->end_data = end_data;
current->mm->start_stack = bprm->p;

mm = current->mm;
mm->end_code = end_code;
mm->start_code = start_code;
mm->start_data = start_data;
mm->end_data = end_data;
mm->start_stack = bprm->p;

if ((current->flags & PF_RANDOMIZE) && (randomize_va_space > 1)) {
/*
Expand All @@ -1111,12 +1115,11 @@ static int load_elf_binary(struct linux_binprm *bprm)
* growing down), and into the unused ELF_ET_DYN_BASE region.
*/
if (IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) &&
elf_ex->e_type == ET_DYN && !interpreter)
current->mm->brk = current->mm->start_brk =
ELF_ET_DYN_BASE;
elf_ex->e_type == ET_DYN && !interpreter) {
mm->brk = mm->start_brk = ELF_ET_DYN_BASE;
}

current->mm->brk = current->mm->start_brk =
arch_randomize_brk(current->mm);
mm->brk = mm->start_brk = arch_randomize_brk(mm);
#ifdef compat_brk_randomized
current->brk_randomized = 1;
#endif
Expand Down Expand Up @@ -1574,14 +1577,15 @@ static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata,
*/
static int fill_files_note(struct memelfnote *note)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned count, size, names_ofs, remaining, n;
user_long_t *data;
user_long_t *start_end_ofs;
char *name_base, *name_curpos;

/* *Estimated* file count and total data size needed */
count = current->mm->map_count;
count = mm->map_count;
if (count > UINT_MAX / 64)
return -EINVAL;
size = count * 64;
Expand All @@ -1599,7 +1603,7 @@ static int fill_files_note(struct memelfnote *note)
name_base = name_curpos = ((char *)data) + names_ofs;
remaining = size - names_ofs;
count = 0;
for (vma = current->mm->mmap; vma != NULL; vma = vma->vm_next) {
for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
struct file *file;
const char *filename;

Expand Down Expand Up @@ -1633,10 +1637,10 @@ static int fill_files_note(struct memelfnote *note)
data[0] = count;
data[1] = PAGE_SIZE;
/*
* Count usually is less than current->mm->map_count,
* Count usually is less than mm->map_count,
* we need to move filenames down.
*/
n = current->mm->map_count - count;
n = mm->map_count - count;
if (n != 0) {
unsigned shift_bytes = n * 3 * sizeof(data[0]);
memmove(name_base - shift_bytes, name_base,
Expand Down

0 comments on commit 03c6d72

Please sign in to comment.