forked from torvalds/linux
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
mm: introduce page_vma_mapped_walk()
Introduce a new interface to check if a page is mapped into a vma. It
aims to address shortcomings of page_check_address{,_transhuge}.
Existing interface is not able to handle PTE-mapped THPs: it only finds
the first PTE. The rest lefted unnoticed.
page_vma_mapped_walk() iterates over all possible mapping of the page in
the vma.
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Andrea Arcangeli <[email protected]>
Cc: Hillf Danton <[email protected]>
Cc: Hugh Dickins <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Oleg Nesterov <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Srikar Dronamraju <[email protected]>
Cc: Vladimir Davydov <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>- Loading branch information
Showing
4 changed files
with
224 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,188 @@ | ||
| #include <linux/mm.h> | ||
| #include <linux/rmap.h> | ||
| #include <linux/hugetlb.h> | ||
| #include <linux/swap.h> | ||
| #include <linux/swapops.h> | ||
|
|
||
| #include "internal.h" | ||
|
|
||
| static inline bool check_pmd(struct page_vma_mapped_walk *pvmw) | ||
| { | ||
| pmd_t pmde; | ||
| /* | ||
| * Make sure we don't re-load pmd between present and !trans_huge check. | ||
| * We need a consistent view. | ||
| */ | ||
| pmde = READ_ONCE(*pvmw->pmd); | ||
| return pmd_present(pmde) && !pmd_trans_huge(pmde); | ||
| } | ||
|
|
||
| static inline bool not_found(struct page_vma_mapped_walk *pvmw) | ||
| { | ||
| page_vma_mapped_walk_done(pvmw); | ||
| return false; | ||
| } | ||
|
|
||
| static bool map_pte(struct page_vma_mapped_walk *pvmw) | ||
| { | ||
| pvmw->pte = pte_offset_map(pvmw->pmd, pvmw->address); | ||
| if (!(pvmw->flags & PVMW_SYNC)) { | ||
| if (pvmw->flags & PVMW_MIGRATION) { | ||
| if (!is_swap_pte(*pvmw->pte)) | ||
| return false; | ||
| } else { | ||
| if (!pte_present(*pvmw->pte)) | ||
| return false; | ||
| } | ||
| } | ||
| pvmw->ptl = pte_lockptr(pvmw->vma->vm_mm, pvmw->pmd); | ||
| spin_lock(pvmw->ptl); | ||
| return true; | ||
| } | ||
|
|
||
| static bool check_pte(struct page_vma_mapped_walk *pvmw) | ||
| { | ||
| if (pvmw->flags & PVMW_MIGRATION) { | ||
| #ifdef CONFIG_MIGRATION | ||
| swp_entry_t entry; | ||
| if (!is_swap_pte(*pvmw->pte)) | ||
| return false; | ||
| entry = pte_to_swp_entry(*pvmw->pte); | ||
| if (!is_migration_entry(entry)) | ||
| return false; | ||
| if (migration_entry_to_page(entry) - pvmw->page >= | ||
| hpage_nr_pages(pvmw->page)) { | ||
| return false; | ||
| } | ||
| if (migration_entry_to_page(entry) < pvmw->page) | ||
| return false; | ||
| #else | ||
| WARN_ON_ONCE(1); | ||
| #endif | ||
| } else { | ||
| if (!pte_present(*pvmw->pte)) | ||
| return false; | ||
|
|
||
| /* THP can be referenced by any subpage */ | ||
| if (pte_page(*pvmw->pte) - pvmw->page >= | ||
| hpage_nr_pages(pvmw->page)) { | ||
| return false; | ||
| } | ||
| if (pte_page(*pvmw->pte) < pvmw->page) | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| /** | ||
| * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at | ||
| * @pvmw->address | ||
| * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags | ||
| * must be set. pmd, pte and ptl must be NULL. | ||
| * | ||
| * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point | ||
| * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is | ||
| * adjusted if needed (for PTE-mapped THPs). | ||
| * | ||
| * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page | ||
| * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in | ||
| * a loop to find all PTEs that map the THP. | ||
| * | ||
| * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry | ||
| * regardless of which page table level the page is mapped at. @pvmw->pmd is | ||
| * NULL. | ||
| * | ||
| * Retruns false if there are no more page table entries for the page in | ||
| * the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped. | ||
| * | ||
| * If you need to stop the walk before page_vma_mapped_walk() returned false, | ||
| * use page_vma_mapped_walk_done(). It will do the housekeeping. | ||
| */ | ||
| bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) | ||
| { | ||
| struct mm_struct *mm = pvmw->vma->vm_mm; | ||
| struct page *page = pvmw->page; | ||
| pgd_t *pgd; | ||
| pud_t *pud; | ||
|
|
||
| /* The only possible pmd mapping has been handled on last iteration */ | ||
| if (pvmw->pmd && !pvmw->pte) | ||
| return not_found(pvmw); | ||
|
|
||
| /* Only for THP, seek to next pte entry makes sense */ | ||
| if (pvmw->pte) { | ||
| if (!PageTransHuge(pvmw->page) || PageHuge(pvmw->page)) | ||
| return not_found(pvmw); | ||
| goto next_pte; | ||
| } | ||
|
|
||
| if (unlikely(PageHuge(pvmw->page))) { | ||
| /* when pud is not present, pte will be NULL */ | ||
| pvmw->pte = huge_pte_offset(mm, pvmw->address); | ||
| if (!pvmw->pte) | ||
| return false; | ||
|
|
||
| pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte); | ||
| spin_lock(pvmw->ptl); | ||
| if (!check_pte(pvmw)) | ||
| return not_found(pvmw); | ||
| return true; | ||
| } | ||
| restart: | ||
| pgd = pgd_offset(mm, pvmw->address); | ||
| if (!pgd_present(*pgd)) | ||
| return false; | ||
| pud = pud_offset(pgd, pvmw->address); | ||
| if (!pud_present(*pud)) | ||
| return false; | ||
| pvmw->pmd = pmd_offset(pud, pvmw->address); | ||
| if (pmd_trans_huge(*pvmw->pmd)) { | ||
| pvmw->ptl = pmd_lock(mm, pvmw->pmd); | ||
| if (!pmd_present(*pvmw->pmd)) | ||
| return not_found(pvmw); | ||
| if (likely(pmd_trans_huge(*pvmw->pmd))) { | ||
| if (pvmw->flags & PVMW_MIGRATION) | ||
| return not_found(pvmw); | ||
| if (pmd_page(*pvmw->pmd) != page) | ||
| return not_found(pvmw); | ||
| return true; | ||
| } else { | ||
| /* THP pmd was split under us: handle on pte level */ | ||
| spin_unlock(pvmw->ptl); | ||
| pvmw->ptl = NULL; | ||
| } | ||
| } else { | ||
| if (!check_pmd(pvmw)) | ||
| return false; | ||
| } | ||
| if (!map_pte(pvmw)) | ||
| goto next_pte; | ||
| while (1) { | ||
| if (check_pte(pvmw)) | ||
| return true; | ||
| next_pte: do { | ||
| pvmw->address += PAGE_SIZE; | ||
| if (pvmw->address >= | ||
| __vma_address(pvmw->page, pvmw->vma) + | ||
| hpage_nr_pages(pvmw->page) * PAGE_SIZE) | ||
| return not_found(pvmw); | ||
| /* Did we cross page table boundary? */ | ||
| if (pvmw->address % PMD_SIZE == 0) { | ||
| pte_unmap(pvmw->pte); | ||
| if (pvmw->ptl) { | ||
| spin_unlock(pvmw->ptl); | ||
| pvmw->ptl = NULL; | ||
| } | ||
| goto restart; | ||
| } else { | ||
| pvmw->pte++; | ||
| } | ||
| } while (pte_none(*pvmw->pte)); | ||
|
|
||
| if (!pvmw->ptl) { | ||
| pvmw->ptl = pte_lockptr(mm, pvmw->pmd); | ||
| spin_lock(pvmw->ptl); | ||
| } | ||
| } | ||
| } |