Skip to content

Commit

Permalink
mm/swap: free swap slots in batch
Browse files Browse the repository at this point in the history 
Add new functions that free unused swap slots in batches without the
need to reacquire swap info lock.  This improves scalability and reduce
lock contention.

Link: http://lkml.kernel.org/r/c25e0fcdfd237ec4ca7db91631d3b9f6ed23824e.1484082593.git.tim.c.chen@linux.intel.com
Signed-off-by: Tim Chen <[email protected]>
Signed-off-by: "Huang, Ying" <[email protected]>
Cc: Aaron Lu <[email protected]>
Cc: Andi Kleen <[email protected]>
Cc: Andrea Arcangeli <[email protected]>
Cc: Christian Borntraeger <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Hillf Danton <[email protected]>
Cc: Huang Ying <[email protected]>
Cc: Hugh Dickins <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Jonathan Corbet <[email protected]> escreveu:
Cc: Kirill A. Shutemov <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Minchan Kim <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Shaohua Li <[email protected]>
Cc: Vladimir Davydov <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@pdxChen @torvalds
pdxChen authored and torvalds committed Feb 23, 2017
1 parent 36005ba commit 7c00baf
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 61 deletions.
1 change: 1 addition & 0 deletions include/linux/swap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ extern int swap_duplicate(swp_entry_t);
extern int swapcache_prepare(swp_entry_t);
extern void swap_free(swp_entry_t);
extern void swapcache_free(swp_entry_t);
extern void swapcache_free_entries(swp_entry_t *entries, int n);
extern int free_swap_and_cache(swp_entry_t);
extern int swap_type_of(dev_t, sector_t, struct block_device **);
extern unsigned int count_swap_pages(int, int);
Expand Down
155 changes: 94 additions & 61 deletions mm/swapfile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -942,35 +942,34 @@ static struct swap_info_struct *swap_info_get(swp_entry_t entry)
return p;
}

static unsigned char swap_entry_free(struct swap_info_struct *p,
swp_entry_t entry, unsigned char usage,
bool swap_info_locked)
static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry,
struct swap_info_struct *q)
{
struct swap_info_struct *p;

p = _swap_info_get(entry);

if (p != q) {
if (q != NULL)
spin_unlock(&q->lock);
if (p != NULL)
spin_lock(&p->lock);
}
return p;
}

static unsigned char __swap_entry_free(struct swap_info_struct *p,
swp_entry_t entry, unsigned char usage)
{
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
unsigned char count;
unsigned char has_cache;
bool lock_swap_info = false;

if (!swap_info_locked) {
count = p->swap_map[offset];
if (!p->cluster_info || count == usage || count == SWAP_MAP_SHMEM) {
lock_swap_info:
swap_info_locked = true;
lock_swap_info = true;
spin_lock(&p->lock);
}
}

ci = lock_cluster(p, offset);
ci = lock_cluster_or_swap_info(p, offset);

count = p->swap_map[offset];

if (!swap_info_locked && (count == usage || count == SWAP_MAP_SHMEM)) {
unlock_cluster(ci);
goto lock_swap_info;
}

has_cache = count & SWAP_HAS_CACHE;
count &= ~SWAP_HAS_CACHE;

Expand All @@ -994,46 +993,52 @@ static unsigned char swap_entry_free(struct swap_info_struct *p,
}

usage = count | has_cache;
p->swap_map[offset] = usage;
p->swap_map[offset] = usage ? : SWAP_HAS_CACHE;

unlock_cluster_or_swap_info(p, ci);

return usage;
}

static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry)
{
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
unsigned char count;

ci = lock_cluster(p, offset);
count = p->swap_map[offset];
VM_BUG_ON(count != SWAP_HAS_CACHE);
p->swap_map[offset] = 0;
dec_cluster_info_page(p, p->cluster_info, offset);
unlock_cluster(ci);

/* free if no reference */
if (!usage) {
VM_BUG_ON(!swap_info_locked);
mem_cgroup_uncharge_swap(entry);
ci = lock_cluster(p, offset);
dec_cluster_info_page(p, p->cluster_info, offset);
unlock_cluster(ci);
if (offset < p->lowest_bit)
p->lowest_bit = offset;
if (offset > p->highest_bit) {
bool was_full = !p->highest_bit;
p->highest_bit = offset;
if (was_full && (p->flags & SWP_WRITEOK)) {
spin_lock(&swap_avail_lock);
WARN_ON(!plist_node_empty(&p->avail_list));
if (plist_node_empty(&p->avail_list))
plist_add(&p->avail_list,
&swap_avail_head);
spin_unlock(&swap_avail_lock);
}
}
atomic_long_inc(&nr_swap_pages);
p->inuse_pages--;
frontswap_invalidate_page(p->type, offset);
if (p->flags & SWP_BLKDEV) {
struct gendisk *disk = p->bdev->bd_disk;
if (disk->fops->swap_slot_free_notify)
disk->fops->swap_slot_free_notify(p->bdev,
offset);
mem_cgroup_uncharge_swap(entry);
if (offset < p->lowest_bit)
p->lowest_bit = offset;
if (offset > p->highest_bit) {
bool was_full = !p->highest_bit;

p->highest_bit = offset;
if (was_full && (p->flags & SWP_WRITEOK)) {
spin_lock(&swap_avail_lock);
WARN_ON(!plist_node_empty(&p->avail_list));
if (plist_node_empty(&p->avail_list))
plist_add(&p->avail_list,
&swap_avail_head);
spin_unlock(&swap_avail_lock);
}
}
atomic_long_inc(&nr_swap_pages);
p->inuse_pages--;
frontswap_invalidate_page(p->type, offset);
if (p->flags & SWP_BLKDEV) {
struct gendisk *disk = p->bdev->bd_disk;

if (lock_swap_info)
spin_unlock(&p->lock);

return usage;
if (disk->fops->swap_slot_free_notify)
disk->fops->swap_slot_free_notify(p->bdev,
offset);
}
}

/*
Expand All @@ -1045,8 +1050,10 @@ void swap_free(swp_entry_t entry)
struct swap_info_struct *p;

p = _swap_info_get(entry);
if (p)
swap_entry_free(p, entry, 1, false);
if (p) {
if (!__swap_entry_free(p, entry, 1))
swapcache_free_entries(&entry, 1);
}
}

/*
Expand All @@ -1057,8 +1064,32 @@ void swapcache_free(swp_entry_t entry)
struct swap_info_struct *p;

p = _swap_info_get(entry);
if (p) {
if (!__swap_entry_free(p, entry, SWAP_HAS_CACHE))
swapcache_free_entries(&entry, 1);
}
}

void swapcache_free_entries(swp_entry_t *entries, int n)
{
struct swap_info_struct *p, *prev;
int i;

if (n <= 0)
return;

prev = NULL;
p = NULL;
for (i = 0; i < n; ++i) {
p = swap_info_get_cont(entries[i], prev);
if (p)
swap_entry_free(p, entries[i]);
else
break;
prev = p;
}
if (p)
swap_entry_free(p, entry, SWAP_HAS_CACHE, false);
spin_unlock(&p->lock);
}

/*
Expand Down Expand Up @@ -1241,21 +1272,23 @@ int free_swap_and_cache(swp_entry_t entry)
{
struct swap_info_struct *p;
struct page *page = NULL;
unsigned char count;

if (non_swap_entry(entry))
return 1;

p = swap_info_get(entry);
p = _swap_info_get(entry);
if (p) {
if (swap_entry_free(p, entry, 1, true) == SWAP_HAS_CACHE) {
count = __swap_entry_free(p, entry, 1);
if (count == SWAP_HAS_CACHE) {
page = find_get_page(swap_address_space(entry),
swp_offset(entry));
if (page && !trylock_page(page)) {
put_page(page);
page = NULL;
}
}
spin_unlock(&p->lock);
} else if (!count)
swapcache_free_entries(&entry, 1);
}
if (page) {
/*
Expand Down

0 comments on commit 7c00baf

Please sign in to comment.