Skip to content

Commit

Permalink
x86/mm: Fix pgd_lock deadlock
Browse files Browse the repository at this point in the history
It's forbidden to take the page_table_lock with the irq disabled
or if there's contention the IPIs (for tlb flushes) sent with
the page_table_lock held will never run leading to a deadlock.

Nobody takes the pgd_lock from irq context so the _irqsave can be
removed.

Signed-off-by: Andrea Arcangeli <[email protected]>
Acked-by: Rik van Riel <[email protected]>
Tested-by: Konrad Rzeszutek Wilk <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: <[email protected]>
LKML-Reference: <[email protected]>
Signed-off-by: Ingo Molnar <[email protected]>
  • Loading branch information
aagit authored and Ingo Molnar committed Mar 10, 2011
1 parent f862685 commit a79e53d
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 30 deletions.
7 changes: 3 additions & 4 deletions arch/x86/mm/fault.c
Original file line number Diff line number Diff line change
Expand Up @@ -229,15 +229,14 @@ void vmalloc_sync_all(void)
for (address = VMALLOC_START & PMD_MASK;
address >= TASK_SIZE && address < FIXADDR_TOP;
address += PMD_SIZE) {

unsigned long flags;
struct page *page;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
list_for_each_entry(page, &pgd_list, lru) {
spinlock_t *pgt_lock;
pmd_t *ret;

/* the pgt_lock only for Xen */
pgt_lock = &pgd_page_get_mm(page)->page_table_lock;

spin_lock(pgt_lock);
Expand All @@ -247,7 +246,7 @@ void vmalloc_sync_all(void)
if (!ret)
break;
}
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}
}

Expand Down
6 changes: 3 additions & 3 deletions arch/x86/mm/init_64.c
Original file line number Diff line number Diff line change
Expand Up @@ -105,18 +105,18 @@ void sync_global_pgds(unsigned long start, unsigned long end)

for (address = start; address <= end; address += PGDIR_SIZE) {
const pgd_t *pgd_ref = pgd_offset_k(address);
unsigned long flags;
struct page *page;

if (pgd_none(*pgd_ref))
continue;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
list_for_each_entry(page, &pgd_list, lru) {
pgd_t *pgd;
spinlock_t *pgt_lock;

pgd = (pgd_t *)page_address(page) + pgd_index(address);
/* the pgt_lock only for Xen */
pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
spin_lock(pgt_lock);

Expand All @@ -128,7 +128,7 @@ void sync_global_pgds(unsigned long start, unsigned long end)

spin_unlock(pgt_lock);
}
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}
}

Expand Down
18 changes: 8 additions & 10 deletions arch/x86/mm/pageattr.c
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,10 @@ static unsigned long direct_pages_count[PG_LEVEL_NUM];

void update_page_count(int level, unsigned long pages)
{
unsigned long flags;

/* Protect against CPA */
spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
direct_pages_count[level] += pages;
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}

static void split_page_count(int level)
Expand Down Expand Up @@ -394,7 +392,7 @@ static int
try_preserve_large_page(pte_t *kpte, unsigned long address,
struct cpa_data *cpa)
{
unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn;
unsigned long nextpage_addr, numpages, pmask, psize, addr, pfn;
pte_t new_pte, old_pte, *tmp;
pgprot_t old_prot, new_prot, req_prot;
int i, do_split = 1;
Expand All @@ -403,7 +401,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
if (cpa->force_split)
return 1;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
/*
* Check for races, another CPU might have split this page
* up already:
Expand Down Expand Up @@ -498,14 +496,14 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
}

out_unlock:
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);

return do_split;
}

static int split_large_page(pte_t *kpte, unsigned long address)
{
unsigned long flags, pfn, pfninc = 1;
unsigned long pfn, pfninc = 1;
unsigned int i, level;
pte_t *pbase, *tmp;
pgprot_t ref_prot;
Expand All @@ -519,7 +517,7 @@ static int split_large_page(pte_t *kpte, unsigned long address)
if (!base)
return -ENOMEM;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
/*
* Check for races, another CPU might have split this page
* up for us already:
Expand Down Expand Up @@ -591,7 +589,7 @@ static int split_large_page(pte_t *kpte, unsigned long address)
*/
if (base)
__free_page(base);
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);

return 0;
}
Expand Down
11 changes: 4 additions & 7 deletions arch/x86/mm/pgtable.c
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,12 @@ static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)

static void pgd_dtor(pgd_t *pgd)
{
unsigned long flags; /* can be called from interrupt context */

if (SHARED_KERNEL_PMD)
return;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);
pgd_list_del(pgd);
spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}

/*
Expand Down Expand Up @@ -260,7 +258,6 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd;
pmd_t *pmds[PREALLOCATED_PMDS];
unsigned long flags;

pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);

Expand All @@ -280,12 +277,12 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
* respect to anything walking the pgd_list, so that they
* never see a partially populated pgd.
*/
spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);

pgd_ctor(mm, pgd);
pgd_prepopulate_pmd(mm, pgd, pmds);

spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);

return pgd;

Expand Down
10 changes: 4 additions & 6 deletions arch/x86/xen/mmu.c
Original file line number Diff line number Diff line change
Expand Up @@ -986,10 +986,9 @@ static void xen_pgd_pin(struct mm_struct *mm)
*/
void xen_mm_pin_all(void)
{
unsigned long flags;
struct page *page;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);

list_for_each_entry(page, &pgd_list, lru) {
if (!PagePinned(page)) {
Expand All @@ -998,7 +997,7 @@ void xen_mm_pin_all(void)
}
}

spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}

/*
Expand Down Expand Up @@ -1099,10 +1098,9 @@ static void xen_pgd_unpin(struct mm_struct *mm)
*/
void xen_mm_unpin_all(void)
{
unsigned long flags;
struct page *page;

spin_lock_irqsave(&pgd_lock, flags);
spin_lock(&pgd_lock);

list_for_each_entry(page, &pgd_list, lru) {
if (PageSavePinned(page)) {
Expand All @@ -1112,7 +1110,7 @@ void xen_mm_unpin_all(void)
}
}

spin_unlock_irqrestore(&pgd_lock, flags);
spin_unlock(&pgd_lock);
}

void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
Expand Down

0 comments on commit a79e53d

Please sign in to comment.