Skip to content

Commit

Permalink
sparc64: add support for folded p4d page tables
Browse files Browse the repository at this point in the history 
Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.

Signed-off-by: Mike Rapoport <[email protected]>
Acked-by: David S. Miller <[email protected]>
Signed-off-by: David S. Miller <[email protected]>
  • Loading branch information
@rppt @davem330
rppt authored and davem330 committed Jan 30, 2020
1 parent 9167bd9 commit 5637bc5
Show file tree
Hide file tree
Showing 7 changed files with 84 additions and 32 deletions.
6 changes: 3 additions & 3 deletions arch/sparc/include/asm/pgalloc_64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@

extern struct kmem_cache *pgtable_cache;

static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
static inline void __p4d_populate(p4d_t *p4d, pud_t *pud)
{
pgd_set(pgd, pud);
p4d_set(p4d, pud);
}

#define pgd_populate(MM, PGD, PUD) __pgd_populate(PGD, PUD)
#define p4d_populate(MM, P4D, PUD) __p4d_populate(P4D, PUD)

static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
Expand Down
24 changes: 12 additions & 12 deletions arch/sparc/include/asm/pgtable_64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* the SpitFire page tables.
*/

#include <asm-generic/5level-fixup.h>
#include <asm-generic/pgtable-nop4d.h>
#include <linux/compiler.h>
#include <linux/const.h>
#include <asm/types.h>
Expand Down Expand Up @@ -810,9 +810,9 @@ static inline int pmd_present(pmd_t pmd)

#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK)

#define pgd_none(pgd) (!pgd_val(pgd))
#define p4d_none(p4d) (!p4d_val(p4d))

#define pgd_bad(pgd) (pgd_val(pgd) & ~PAGE_MASK)
#define p4d_bad(p4d) (p4d_val(p4d) & ~PAGE_MASK)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
Expand Down Expand Up @@ -859,13 +859,13 @@ static inline unsigned long pud_page_vaddr(pud_t pud)
#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
#define pud_present(pud) (pud_val(pud) != 0U)
#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
#define pgd_page_vaddr(pgd) \
((unsigned long) __va(pgd_val(pgd)))
#define pgd_present(pgd) (pgd_val(pgd) != 0U)
#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL)
#define p4d_page_vaddr(p4d) \
((unsigned long) __va(p4d_val(p4d)))
#define p4d_present(p4d) (p4d_val(p4d) != 0U)
#define p4d_clear(p4dp) (p4d_val(*(p4dp)) = 0UL)

/* only used by the stubbed out hugetlb gup code, should never be called */
#define pgd_page(pgd) NULL
#define p4d_page(p4d) NULL

static inline unsigned long pud_large(pud_t pud)
{
Expand All @@ -884,8 +884,8 @@ static inline unsigned long pud_pfn(pud_t pud)
/* Same in both SUN4V and SUN4U. */
#define pte_none(pte) (!pte_val(pte))

#define pgd_set(pgdp, pudp) \
(pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
#define p4d_set(p4dp, pudp) \
(p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp))))

/* to find an entry in a page-table-directory. */
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
Expand All @@ -896,8 +896,8 @@ static inline unsigned long pud_pfn(pud_t pud)

/* Find an entry in the third-level page table.. */
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
#define pud_offset(pgdp, address) \
((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
#define pud_offset(p4dp, address) \
((pud_t *) p4d_page_vaddr(*(p4dp)) + pud_index(address))

/* Find an entry in the second-level page table.. */
#define pmd_offset(pudp, address) \
Expand Down
6 changes: 5 additions & 1 deletion arch/sparc/kernel/signal32.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ static void flush_signal_insns(unsigned long address)
unsigned long pstate, paddr;
pte_t *ptep, pte;
pgd_t *pgdp;
p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;

Expand All @@ -318,7 +319,10 @@ static void flush_signal_insns(unsigned long address)
pgdp = pgd_offset(current->mm, address);
if (pgd_none(*pgdp))
goto out_irqs_on;
pudp = pud_offset(pgdp, address);
p4dp = p4d_offset(pgdp, address);
if (p4d_none(*p4dp))
goto out_irqs_on;
pudp = pud_offset(p4dp, address);
if (pud_none(*pudp))
goto out_irqs_on;
pmdp = pmd_offset(pudp, address);
Expand Down
13 changes: 12 additions & 1 deletion arch/sparc/kernel/smp_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1621,6 +1621,7 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
static void __init pcpu_populate_pte(unsigned long addr)
{
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;

Expand All @@ -1633,7 +1634,17 @@ static void __init pcpu_populate_pte(unsigned long addr)
pgd_populate(&init_mm, pgd, new);
}

pud = pud_offset(pgd, addr);
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) {
pud_t *new;

new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
if (!new)
goto err_alloc;
p4d_populate(&init_mm, p4d, new);
}

pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
pmd_t *new;

Expand Down
6 changes: 5 additions & 1 deletion arch/sparc/mm/fault_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
static unsigned int get_user_insn(unsigned long tpc)
{
pgd_t *pgdp = pgd_offset(current->mm, tpc);
p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep, pte;
Expand All @@ -88,7 +89,10 @@ static unsigned int get_user_insn(unsigned long tpc)

if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
goto out;
pudp = pud_offset(pgdp, tpc);
p4dp = p4d_offset(pgdp, tpc);
if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
goto out;
pudp = pud_offset(p4dp, tpc);
if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
goto out;

Expand Down
28 changes: 18 additions & 10 deletions arch/sparc/mm/hugetlbpage.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,11 +277,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
unsigned long addr, unsigned long sz)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;

pgd = pgd_offset(mm, addr);
pud = pud_alloc(mm, pgd, addr);
p4d = p4d_offset(pgd, addr);
pud = pud_alloc(mm, p4d, addr);
if (!pud)
return NULL;
if (sz >= PUD_SIZE)
Expand All @@ -298,13 +300,17 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
unsigned long addr, unsigned long sz)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;

pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd))
return NULL;
pud = pud_offset(pgd, addr);
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d))
return NULL;
pud = pud_offset(p4d, addr);
if (pud_none(*pud))
return NULL;
if (is_hugetlb_pud(*pud))
Expand Down Expand Up @@ -449,7 +455,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
mm_dec_nr_pmds(tlb->mm);
}

static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
Expand All @@ -458,7 +464,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
unsigned long start;

start = addr;
pud = pud_offset(pgd, addr);
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
Expand All @@ -481,8 +487,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
if (end - 1 > ceiling - 1)
return;

pud = pud_offset(pgd, start);
pgd_clear(pgd);
pud = pud_offset(p4d, start);
p4d_clear(p4d);
pud_free_tlb(tlb, pud, start);
mm_dec_nr_puds(tlb->mm);
}
Expand All @@ -492,6 +498,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
unsigned long floor, unsigned long ceiling)
{
pgd_t *pgd;
p4d_t *p4d;
unsigned long next;

addr &= PMD_MASK;
Expand All @@ -511,10 +518,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
return;

pgd = pgd_offset(tlb->mm, addr);
p4d = p4d_offset(pgd, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(p4d))
continue;
hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
} while (pgd++, addr = next, addr != end);
hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
} while (p4d++, addr = next, addr != end);
}
33 changes: 29 additions & 4 deletions arch/sparc/mm/init_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,8 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end)
paddr = kaddr & mask;
else {
pgd_t *pgdp = pgd_offset_k(kaddr);
pud_t *pudp = pud_offset(pgdp, kaddr);
p4d_t *p4dp = p4d_offset(pgdp, kaddr);
pud_t *pudp = pud_offset(p4dp, kaddr);
pmd_t *pmdp = pmd_offset(pudp, kaddr);
pte_t *ptep = pte_offset_kernel(pmdp, kaddr);

Expand Down Expand Up @@ -1653,6 +1654,7 @@ static unsigned long max_phys_bits = 40;
bool kern_addr_valid(unsigned long addr)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
Expand All @@ -1674,7 +1676,11 @@ bool kern_addr_valid(unsigned long addr)
if (pgd_none(*pgd))
return 0;

pud = pud_offset(pgd, addr);
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d))
return 0;

pud = pud_offset(p4d, addr);
if (pud_none(*pud))
return 0;

Expand Down Expand Up @@ -1800,6 +1806,7 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
while (vstart < vend) {
unsigned long this_end, paddr = __pa(vstart);
pgd_t *pgd = pgd_offset_k(vstart);
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
Expand All @@ -1814,7 +1821,20 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
alloc_bytes += PAGE_SIZE;
pgd_populate(&init_mm, pgd, new);
}
pud = pud_offset(pgd, vstart);

p4d = p4d_offset(pgd, vstart);
if (p4d_none(*p4d)) {
pud_t *new;

new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
PAGE_SIZE);
if (!new)
goto err_alloc;
alloc_bytes += PAGE_SIZE;
p4d_populate(&init_mm, p4d, new);
}

pud = pud_offset(p4d, vstart);
if (pud_none(*pud)) {
pmd_t *new;

Expand Down Expand Up @@ -2612,13 +2632,18 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
for (; vstart < vend; vstart += PMD_SIZE) {
pgd_t *pgd = vmemmap_pgd_populate(vstart, node);
unsigned long pte;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;

if (!pgd)
return -ENOMEM;

pud = vmemmap_pud_populate(pgd, vstart, node);
p4d = vmemmap_p4d_populate(pgd, vstart, node);
if (!p4d)
return -ENOMEM;

pud = vmemmap_pud_populate(p4d, vstart, node);
if (!pud)
return -ENOMEM;

Expand Down

0 comments on commit 5637bc5

Please sign in to comment.