Skip to content

Commit

Permalink
mm: hugetlbfs: move the put/get_page slab and hugetlbfs optimization …
Browse files Browse the repository at this point in the history 
…in a faster path

We don't actually need a reference on the head page in the slab and
hugetlbfs paths, as long as we add a smp_rmb() which should be faster
than get_page_unless_zero.

[[email protected]: fix typo in comment]
Signed-off-by: Andrea Arcangeli <[email protected]>
Cc: Khalid Aziz <[email protected]>
Cc: Pravin Shelar <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Ben Hutchings <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Andi Kleen <[email protected]>
Cc: Minchan Kim <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@aagit @torvalds
aagit authored and torvalds committed Jan 22, 2014
1 parent a0368d4 commit ebf360f
Showing 1 changed file with 78 additions and 62 deletions.
140 changes: 78 additions & 62 deletions mm/swap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,45 +86,61 @@ static void put_compound_page(struct page *page)
/* __split_huge_page_refcount can run under us */
struct page *page_head = compound_trans_head(page);

/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In
* particular slab_unlock() in slub used to be a hot
* path. It is still hot on arches that do not support
* this_cpu_cmpxchg_double().
*
* If "page" is part of a slab or hugetlbfs page it
* cannot be splitted and the head page cannot change
* from under us. And if "page" is part of a THP page
* under splitting, if the head page pointed by the
* THP tail isn't a THP head anymore, we'll find
* PageTail clear after smp_rmb() and we'll treat it
* as a single page.
*/
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
/*
* If "page" is a THP tail, we must read the tail page
* flags after the head page flags. The
* split_huge_page side enforces write memory
* barriers between clearing PageTail and before the
* head page can be freed and reallocated.
*/
smp_rmb();
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
VM_BUG_ON(page_mapcount(page) <= 0);
atomic_dec(&page->_mapcount);
if (put_page_testzero(page_head))
__put_compound_page(page_head);
return;
} else
/*
* __split_huge_page_refcount
* run before us, "page" was a
* THP tail. The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order (only possible if
* reallocated as slab on
* x86).
*/
goto out_put_single;
}

if (likely(page != page_head &&
get_page_unless_zero(page_head))) {
unsigned long flags;

/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is
* still hot on arches that do not support
* this_cpu_cmpxchg_double().
*/
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
atomic_dec(&page->_mapcount);
if (put_page_testzero(page_head))
VM_BUG_ON(1);
if (put_page_testzero(page_head))
__put_compound_page(page_head);
return;
} else
/*
* __split_huge_page_refcount
* run before us, "page" was a
* THP tail. The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order (only possible if
* reallocated as slab on
* x86).
*/
goto skip_lock;
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
Expand All @@ -135,7 +151,6 @@ static void put_compound_page(struct page *page)
if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags);
skip_lock:
if (put_page_testzero(page_head)) {
/*
* The head page may have been
Expand Down Expand Up @@ -221,36 +236,37 @@ bool __get_page_tail(struct page *page)
* split_huge_page().
*/
unsigned long flags;
bool got = false;
bool got;
struct page *page_head = compound_trans_head(page);

if (likely(page != page_head && get_page_unless_zero(page_head))) {
/* Ref to put_compound_page() comment. */
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
if (likely(PageTail(page))) {
/*
* This is a hugetlbfs page or a slab
* page. __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
__get_page_tail_foll(page, false);
return true;
} else {
/*
* __split_huge_page_refcount run
* before us, "page" was a THP
* tail. The split page_head has been
* freed and reallocated as slab or
* hugetlbfs page of smaller order
* (only possible if reallocated as
* slab on x86).
*/
put_page(page_head);
return false;
}
/* Ref to put_compound_page() comment. */
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
smp_rmb();
if (likely(PageTail(page))) {
/*
* This is a hugetlbfs page or a slab
* page. __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
__get_page_tail_foll(page, true);
return true;
} else {
/*
* __split_huge_page_refcount run
* before us, "page" was a THP
* tail. The split page_head has been
* freed and reallocated as slab or
* hugetlbfs page of smaller order
* (only possible if reallocated as
* slab on x86).
*/
return false;
}
}

got = false;
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
Expand Down

0 comments on commit ebf360f

Please sign in to comment.