Skip to content

Commit

Permalink
mm: add account_locked_vm utility function
Browse files Browse the repository at this point in the history 
locked_vm accounting is done roughly the same way in five places, so
unify them in a helper.

Include the helper's caller in the debug print to distinguish between
callsites.

Error codes stay the same, so user-visible behavior does too.  The one
exception is that the -EPERM case in tce_account_locked_vm is removed
because Alexey has never seen it triggered.

[[email protected]: v3]
  Link: http://lkml.kernel.org/r/[email protected]
[[email protected]: fix mm/util.c]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Daniel Jordan <[email protected]>
Signed-off-by: Stephen Rothwell <[email protected]>
Tested-by: Alexey Kardashevskiy <[email protected]>
Acked-by: Alex Williamson <[email protected]>
Cc: Alan Tull <[email protected]>
Cc: Alex Williamson <[email protected]>
Cc: Benjamin Herrenschmidt <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: Christophe Leroy <[email protected]>
Cc: Davidlohr Bueso <[email protected]>
Cc: Jason Gunthorpe <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Michael Ellerman <[email protected]>
Cc: Moritz Fischer <[email protected]>
Cc: Paul Mackerras <[email protected]>
Cc: Steve Sistare <[email protected]>
Cc: Wu Hao <[email protected]>
Cc: Ira Weiny <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@danieljordan10 @torvalds
danieljordan10 authored and torvalds committed Jul 17, 2019
1 parent 73b20c8 commit 79eb597
Show file tree
Hide file tree
Showing 7 changed files with 98 additions and 190 deletions.
44 changes: 4 additions & 40 deletions arch/powerpc/kvm/book3s_64_vio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <linux/anon_inodes.h>
#include <linux/iommu.h>
#include <linux/file.h>
#include <linux/mm.h>

#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
Expand All @@ -45,43 +46,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
}

static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
{
long ret = 0;

if (!current || !current->mm)
return ret; /* process exited */

down_write(&current->mm->mmap_sem);

if (inc) {
unsigned long locked, lock_limit;

locked = current->mm->locked_vm + stt_pages;
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (locked > lock_limit && !capable(CAP_IPC_LOCK))
ret = -ENOMEM;
else
current->mm->locked_vm += stt_pages;
} else {
if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
stt_pages = current->mm->locked_vm;

current->mm->locked_vm -= stt_pages;
}

pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
inc ? '+' : '-',
stt_pages << PAGE_SHIFT,
current->mm->locked_vm << PAGE_SHIFT,
rlimit(RLIMIT_MEMLOCK),
ret ? " - exceeded" : "");

up_write(&current->mm->mmap_sem);

return ret;
}

static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
{
struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
Expand Down Expand Up @@ -291,7 +255,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)

kvm_put_kvm(stt->kvm);

kvmppc_account_memlimit(
account_locked_vm(current->mm,
kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
call_rcu(&stt->rcu, release_spapr_tce_table);

Expand All @@ -316,7 +280,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
return -EINVAL;

npages = kvmppc_tce_pages(size);
ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
if (ret)
return ret;

Expand Down Expand Up @@ -362,7 +326,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,

kfree(stt);
fail_acct:
kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
return ret;
}

Expand Down
41 changes: 4 additions & 37 deletions arch/powerpc/mm/book3s64/iommu_api.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/sizes.h>
#include <linux/mm.h>
#include <asm/mmu_context.h>
#include <asm/pte-walk.h>
#include <linux/mm_inline.h>
Expand Down Expand Up @@ -46,40 +47,6 @@ struct mm_iommu_table_group_mem_t {
u64 dev_hpa; /* Device memory base address */
};

static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
unsigned long npages, bool incr)
{
long ret = 0, locked, lock_limit;

if (!npages)
return 0;

down_write(&mm->mmap_sem);

if (incr) {
locked = mm->locked_vm + npages;
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (locked > lock_limit && !capable(CAP_IPC_LOCK))
ret = -ENOMEM;
else
mm->locked_vm += npages;
} else {
if (WARN_ON_ONCE(npages > mm->locked_vm))
npages = mm->locked_vm;
mm->locked_vm -= npages;
}

pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
current ? current->pid : 0,
incr ? '+' : '-',
npages << PAGE_SHIFT,
mm->locked_vm << PAGE_SHIFT,
rlimit(RLIMIT_MEMLOCK));
up_write(&mm->mmap_sem);

return ret;
}

bool mm_iommu_preregistered(struct mm_struct *mm)
{
return !list_empty(&mm->context.iommu_group_mem_list);
Expand All @@ -96,7 +63,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
unsigned long entry, chunk;

if (dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
ret = mm_iommu_adjust_locked_vm(mm, entries, true);
ret = account_locked_vm(mm, entries, true);
if (ret)
return ret;

Expand Down Expand Up @@ -211,7 +178,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
kfree(mem);

unlock_exit:
mm_iommu_adjust_locked_vm(mm, locked_entries, false);
account_locked_vm(mm, locked_entries, false);

return ret;
}
Expand Down Expand Up @@ -311,7 +278,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
unlock_exit:
mutex_unlock(&mem_list_mutex);

mm_iommu_adjust_locked_vm(mm, unlock_entries, false);
account_locked_vm(mm, unlock_entries, false);

return ret;
}
Expand Down
53 changes: 4 additions & 49 deletions drivers/fpga/dfl-afu-dma-region.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <linux/dma-mapping.h>
#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/mm.h>

#include "dfl-afu.h"

Expand All @@ -31,52 +32,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
afu->dma_regions = RB_ROOT;
}

/**
* afu_dma_adjust_locked_vm - adjust locked memory
* @dev: port device
* @npages: number of pages
* @incr: increase or decrease locked memory
*
* Increase or decrease the locked memory size with npages input.
*
* Return 0 on success.
* Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK.
*/
static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr)
{
unsigned long locked, lock_limit;
int ret = 0;

/* the task is exiting. */
if (!current->mm)
return 0;

down_write(&current->mm->mmap_sem);

if (incr) {
locked = current->mm->locked_vm + npages;
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

if (locked > lock_limit && !capable(CAP_IPC_LOCK))
ret = -ENOMEM;
else
current->mm->locked_vm += npages;
} else {
if (WARN_ON_ONCE(npages > current->mm->locked_vm))
npages = current->mm->locked_vm;
current->mm->locked_vm -= npages;
}

dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid,
incr ? '+' : '-', npages << PAGE_SHIFT,
current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK),
ret ? "- exceeded" : "");

up_write(&current->mm->mmap_sem);

return ret;
}

/**
* afu_dma_pin_pages - pin pages of given dma memory region
* @pdata: feature device platform data
Expand All @@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
struct device *dev = &pdata->dev->dev;
int ret, pinned;

ret = afu_dma_adjust_locked_vm(dev, npages, true);
ret = account_locked_vm(current->mm, npages, true);
if (ret)
return ret;

Expand Down Expand Up @@ -121,7 +76,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
free_pages:
kfree(region->pages);
unlock_vm:
afu_dma_adjust_locked_vm(dev, npages, false);
account_locked_vm(current->mm, npages, false);
return ret;
}

Expand All @@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,

put_all_pages(region->pages, npages);
kfree(region->pages);
afu_dma_adjust_locked_vm(dev, npages, false);
account_locked_vm(current->mm, npages, false);

dev_dbg(dev, "%ld pages unpinned\n", npages);
}
Expand Down
54 changes: 5 additions & 49 deletions drivers/vfio/vfio_iommu_spapr_tce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <linux/vmalloc.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/mm.h>

#include <asm/iommu.h>
#include <asm/tce.h>
Expand All @@ -31,51 +32,6 @@
static void tce_iommu_detach_group(void *iommu_data,
struct iommu_group *iommu_group);

static long try_increment_locked_vm(struct mm_struct *mm, long npages)
{
long ret = 0, locked, lock_limit;

if (WARN_ON_ONCE(!mm))
return -EPERM;

if (!npages)
return 0;

down_write(&mm->mmap_sem);
locked = mm->locked_vm + npages;
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (locked > lock_limit && !capable(CAP_IPC_LOCK))
ret = -ENOMEM;
else
mm->locked_vm += npages;

pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid,
npages << PAGE_SHIFT,
mm->locked_vm << PAGE_SHIFT,
rlimit(RLIMIT_MEMLOCK),
ret ? " - exceeded" : "");

up_write(&mm->mmap_sem);

return ret;
}

static void decrement_locked_vm(struct mm_struct *mm, long npages)
{
if (!mm || !npages)
return;

down_write(&mm->mmap_sem);
if (WARN_ON_ONCE(npages > mm->locked_vm))
npages = mm->locked_vm;
mm->locked_vm -= npages;
pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid,
npages << PAGE_SHIFT,
mm->locked_vm << PAGE_SHIFT,
rlimit(RLIMIT_MEMLOCK));
up_write(&mm->mmap_sem);
}

/*
* VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
*
Expand Down Expand Up @@ -333,7 +289,7 @@ static int tce_iommu_enable(struct tce_container *container)
return ret;

locked = table_group->tce32_size >> PAGE_SHIFT;
ret = try_increment_locked_vm(container->mm, locked);
ret = account_locked_vm(container->mm, locked, true);
if (ret)
return ret;

Expand All @@ -352,7 +308,7 @@ static void tce_iommu_disable(struct tce_container *container)
container->enabled = false;

BUG_ON(!container->mm);
decrement_locked_vm(container->mm, container->locked_pages);
account_locked_vm(container->mm, container->locked_pages, false);
}

static void *tce_iommu_open(unsigned long arg)
Expand Down Expand Up @@ -656,7 +612,7 @@ static long tce_iommu_create_table(struct tce_container *container,
if (!table_size)
return -EINVAL;

ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT);
ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
if (ret)
return ret;

Expand All @@ -675,7 +631,7 @@ static void tce_iommu_free_table(struct tce_container *container,
unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;

iommu_tce_table_put(tbl);
decrement_locked_vm(container->mm, pages);
account_locked_vm(container->mm, pages, false);
}

static long tce_iommu_create_window(struct tce_container *container,
Expand Down
17 changes: 2 additions & 15 deletions drivers/vfio/vfio_iommu_type1.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,21 +272,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)

ret = down_write_killable(&mm->mmap_sem);
if (!ret) {
if (npage > 0) {
if (!dma->lock_cap) {
unsigned long limit;

limit = task_rlimit(dma->task,
RLIMIT_MEMLOCK) >> PAGE_SHIFT;

if (mm->locked_vm + npage > limit)
ret = -ENOMEM;
}
}

if (!ret)
mm->locked_vm += npage;

ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
dma->lock_cap);
up_write(&mm->mmap_sem);
}

Expand Down
4 changes: 4 additions & 0 deletions include/linux/mm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1543,6 +1543,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
int get_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages);

int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
struct task_struct *task, bool bypass_rlim);

/* Container for pinned pfns / pages */
struct frame_vector {
unsigned int nr_allocated; /* Number of frames we have space for */
Expand Down
Loading

0 comments on commit 79eb597

Please sign in to comment.