Skip to content

Commit dd1842a

Browse files
kvaneeshmpe
kvaneesh
authored and
mpe
committed
powerpc/mm: Make page table size a variable
Radix and hash MMU models support different page table sizes. Make the #defines a variable so that existing code can work with variable sizes. Slice related code is only used by hash, so use hash constants there. We will replicate some of the boundary conditions with resepct to TASK_SIZE using radix values too. Right now we do boundary condition check using hash constants. Swapper pgdir size is initialized in asm code. We select the max pgd size to keep it simple. For now we select hash pgdir. When adding radix we will switch that to radix pgdir which is 64K. BUILD_BUG_ON check which is removed is already done in hugepage_init() using MAYBE_BUILD_BUG_ON(). Signed-off-by: Aneesh Kumar K.V <[email protected]> Signed-off-by: Michael Ellerman <[email protected]>
1 parent 13f829a commit dd1842a

14 files changed

+135
-97
lines changed

‎arch/powerpc/include/asm/book3s/64/hash-4k.h

+8-37
Original file line numberDiff line numberDiff line change
@@ -5,48 +5,20 @@
55
* for each page table entry. The PMD and PGD level use a 32b record for
66
* each entry by assuming that each entry is page aligned.
77
*/
8-
#define PTE_INDEX_SIZE 9
9-
#define PMD_INDEX_SIZE 7
10-
#define PUD_INDEX_SIZE 9
11-
#define PGD_INDEX_SIZE 9
8+
#define H_PTE_INDEX_SIZE 9
9+
#define H_PMD_INDEX_SIZE 7
10+
#define H_PUD_INDEX_SIZE 9
11+
#define H_PGD_INDEX_SIZE 9
1212

1313
#ifndef __ASSEMBLY__
14-
#define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_INDEX_SIZE)
15-
#define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
16-
#define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
17-
#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
18-
#endif /* __ASSEMBLY__ */
19-
20-
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
21-
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
22-
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
23-
#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
24-
25-
/* PMD_SHIFT determines what a second-level page table entry can map */
26-
#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
27-
#define PMD_SIZE (1UL << PMD_SHIFT)
28-
#define PMD_MASK (~(PMD_SIZE-1))
14+
#define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE)
15+
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << H_PMD_INDEX_SIZE)
16+
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE)
17+
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE)
2918

3019
/* With 4k base page size, hugepage PTEs go at the PMD level */
3120
#define MIN_HUGEPTE_SHIFT PMD_SHIFT
3221

33-
/* PUD_SHIFT determines what a third-level page table entry can map */
34-
#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
35-
#define PUD_SIZE (1UL << PUD_SHIFT)
36-
#define PUD_MASK (~(PUD_SIZE-1))
37-
38-
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
39-
#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
40-
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
41-
#define PGDIR_MASK (~(PGDIR_SIZE-1))
42-
43-
/* Bits to mask out from a PMD to get to the PTE page */
44-
#define PMD_MASKED_BITS 0
45-
/* Bits to mask out from a PUD to get to the PMD page */
46-
#define PUD_MASKED_BITS 0
47-
/* Bits to mask out from a PGD to get to the PUD page */
48-
#define PGD_MASKED_BITS 0
49-
5022
/* PTE flags to conserve for HPTE identification */
5123
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
5224
H_PAGE_F_SECOND | H_PAGE_F_GIX)
@@ -56,7 +28,6 @@
5628
#define H_PAGE_4K_PFN 0x0
5729
#define H_PAGE_THP_HUGE 0x0
5830
#define H_PAGE_COMBO 0x0
59-
#ifndef __ASSEMBLY__
6031
/*
6132
* On all 4K setups, remap_4k_pfn() equates to remap_pfn_range()
6233
*/

‎arch/powerpc/include/asm/book3s/64/hash-64k.h

+10-36
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,14 @@
11
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H
22
#define _ASM_POWERPC_BOOK3S_64_HASH_64K_H
33

4-
#define PTE_INDEX_SIZE 8
5-
#define PMD_INDEX_SIZE 5
6-
#define PUD_INDEX_SIZE 5
7-
#define PGD_INDEX_SIZE 12
8-
9-
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
10-
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
11-
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
12-
#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
4+
#define H_PTE_INDEX_SIZE 8
5+
#define H_PMD_INDEX_SIZE 5
6+
#define H_PUD_INDEX_SIZE 5
7+
#define H_PGD_INDEX_SIZE 12
138

149
/* With 4k base page size, hugepage PTEs go at the PMD level */
1510
#define MIN_HUGEPTE_SHIFT PAGE_SHIFT
1611

17-
/* PMD_SHIFT determines what a second-level page table entry can map */
18-
#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
19-
#define PMD_SIZE (1UL << PMD_SHIFT)
20-
#define PMD_MASK (~(PMD_SIZE-1))
21-
22-
/* PUD_SHIFT determines what a third-level page table entry can map */
23-
#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
24-
#define PUD_SIZE (1UL << PUD_SHIFT)
25-
#define PUD_MASK (~(PUD_SIZE-1))
26-
27-
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
28-
#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
29-
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
30-
#define PGDIR_MASK (~(PGDIR_SIZE-1))
31-
3212
#define H_PAGE_COMBO 0x00001000 /* this is a combo 4k page */
3313
#define H_PAGE_4K_PFN 0x00002000 /* PFN is for a single 4k page */
3414
/*
@@ -57,13 +37,6 @@
5737
#define PTE_FRAG_SIZE_SHIFT 12
5838
#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
5939

60-
/* Bits to mask out from a PMD to get to the PTE page */
61-
#define PMD_MASKED_BITS 0xc0000000000000ffUL
62-
/* Bits to mask out from a PUD to get to the PMD page */
63-
#define PUD_MASKED_BITS 0xc0000000000000ffUL
64-
/* Bits to mask out from a PGD to get to the PUD page */
65-
#define PGD_MASKED_BITS 0xc0000000000000ffUL
66-
6740
#ifndef __ASSEMBLY__
6841
#include <asm/errno.h>
6942

@@ -135,14 +108,15 @@ static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
135108
__pgprot(pgprot_val(prot) | H_PAGE_4K_PFN));
136109
}
137110

138-
#define PTE_TABLE_SIZE PTE_FRAG_SIZE
111+
#define H_PTE_TABLE_SIZE PTE_FRAG_SIZE
139112
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
140-
#define PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + (sizeof(unsigned long) << PMD_INDEX_SIZE))
113+
#define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
114+
(sizeof(unsigned long) << PMD_INDEX_SIZE))
141115
#else
142-
#define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
116+
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
143117
#endif
144-
#define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
145-
#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
118+
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
119+
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
146120

147121
#ifdef CONFIG_HUGETLB_PAGE
148122
/*

‎arch/powerpc/include/asm/book3s/64/hash.h

+9-5
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,18 @@
2929
/*
3030
* Size of EA range mapped by our pagetables.
3131
*/
32-
#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
33-
PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)
34-
#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE)
32+
#define H_PGTABLE_EADDR_SIZE (H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + \
33+
H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT)
34+
#define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE)
3535

3636
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
37-
#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1)
37+
/*
38+
* only with hash we need to use the second half of pmd page table
39+
* to store pointer to deposited pgtable_t
40+
*/
41+
#define H_PMD_CACHE_INDEX (H_PMD_INDEX_SIZE + 1)
3842
#else
39-
#define PMD_CACHE_INDEX PMD_INDEX_SIZE
43+
#define H_PMD_CACHE_INDEX H_PMD_INDEX_SIZE
4044
#endif
4145
/*
4246
* Define the address range of the kernel non-linear virtual area

‎arch/powerpc/include/asm/book3s/64/mmu-hash.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ extern void slb_set_size(u16 size);
462462
add rt,rt,rx
463463

464464
/* 4 bits per slice and we have one slice per 1TB */
465-
#define SLICE_ARRAY_SIZE (PGTABLE_RANGE >> 41)
465+
#define SLICE_ARRAY_SIZE (H_PGTABLE_RANGE >> 41)
466466

467467
#ifndef __ASSEMBLY__
468468

@@ -533,7 +533,7 @@ static inline unsigned long get_vsid(unsigned long context, unsigned long ea,
533533
/*
534534
* Bad address. We return VSID 0 for that
535535
*/
536-
if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
536+
if ((ea & ~REGION_MASK) >= H_PGTABLE_RANGE)
537537
return 0;
538538

539539
if (ssize == MMU_SEGSIZE_256M)

‎arch/powerpc/include/asm/book3s/64/pgtable.h

+60
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,66 @@
144144
#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
145145
#define PAGE_AGP (PAGE_KERNEL_NC)
146146

147+
#ifndef __ASSEMBLY__
148+
/*
149+
* page table defines
150+
*/
151+
extern unsigned long __pte_index_size;
152+
extern unsigned long __pmd_index_size;
153+
extern unsigned long __pud_index_size;
154+
extern unsigned long __pgd_index_size;
155+
extern unsigned long __pmd_cache_index;
156+
#define PTE_INDEX_SIZE __pte_index_size
157+
#define PMD_INDEX_SIZE __pmd_index_size
158+
#define PUD_INDEX_SIZE __pud_index_size
159+
#define PGD_INDEX_SIZE __pgd_index_size
160+
#define PMD_CACHE_INDEX __pmd_cache_index
161+
/*
162+
* Because of use of pte fragments and THP, size of page table
163+
* are not always derived out of index size above.
164+
*/
165+
extern unsigned long __pte_table_size;
166+
extern unsigned long __pmd_table_size;
167+
extern unsigned long __pud_table_size;
168+
extern unsigned long __pgd_table_size;
169+
#define PTE_TABLE_SIZE __pte_table_size
170+
#define PMD_TABLE_SIZE __pmd_table_size
171+
#define PUD_TABLE_SIZE __pud_table_size
172+
#define PGD_TABLE_SIZE __pgd_table_size
173+
/*
174+
* Pgtable size used by swapper, init in asm code
175+
* We will switch this later to radix PGD
176+
*/
177+
#define MAX_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE)
178+
179+
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
180+
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
181+
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
182+
#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
183+
184+
/* PMD_SHIFT determines what a second-level page table entry can map */
185+
#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
186+
#define PMD_SIZE (1UL << PMD_SHIFT)
187+
#define PMD_MASK (~(PMD_SIZE-1))
188+
189+
/* PUD_SHIFT determines what a third-level page table entry can map */
190+
#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
191+
#define PUD_SIZE (1UL << PUD_SHIFT)
192+
#define PUD_MASK (~(PUD_SIZE-1))
193+
194+
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
195+
#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
196+
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
197+
#define PGDIR_MASK (~(PGDIR_SIZE-1))
198+
199+
/* Bits to mask out from a PMD to get to the PTE page */
200+
#define PMD_MASKED_BITS 0xc0000000000000ffUL
201+
/* Bits to mask out from a PUD to get to the PMD page */
202+
#define PUD_MASKED_BITS 0xc0000000000000ffUL
203+
/* Bits to mask out from a PGD to get to the PUD page */
204+
#define PGD_MASKED_BITS 0xc0000000000000ffUL
205+
#endif /* __ASSEMBLY__ */
206+
147207
#include <asm/book3s/64/hash.h>
148208
#include <asm/barrier.h>
149209

‎arch/powerpc/include/asm/page_64.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ extern u64 ppc64_pft_size;
9393

9494
#define SLICE_LOW_TOP (0x100000000ul)
9595
#define SLICE_NUM_LOW (SLICE_LOW_TOP >> SLICE_LOW_SHIFT)
96-
#define SLICE_NUM_HIGH (PGTABLE_RANGE >> SLICE_HIGH_SHIFT)
96+
#define SLICE_NUM_HIGH (H_PGTABLE_RANGE >> SLICE_HIGH_SHIFT)
9797

9898
#define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT)
9999
#define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT)

‎arch/powerpc/kernel/asm-offsets.c

+4
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,11 @@ int main(void)
438438
DEFINE(BUG_ENTRY_SIZE, sizeof(struct bug_entry));
439439
#endif
440440

441+
#ifdef MAX_PGD_TABLE_SIZE
442+
DEFINE(PGD_TABLE_SIZE, MAX_PGD_TABLE_SIZE);
443+
#else
441444
DEFINE(PGD_TABLE_SIZE, PGD_TABLE_SIZE);
445+
#endif
442446
DEFINE(PTE_SIZE, sizeof(pte_t));
443447

444448
#ifdef CONFIG_KVM

‎arch/powerpc/mm/hash_utils_64.c

+12
Original file line numberDiff line numberDiff line change
@@ -870,6 +870,18 @@ static void __init htab_initialize(void)
870870

871871
void __init early_init_mmu(void)
872872
{
873+
/*
874+
* initialize page table size
875+
*/
876+
__pte_index_size = H_PTE_INDEX_SIZE;
877+
__pmd_index_size = H_PMD_INDEX_SIZE;
878+
__pud_index_size = H_PUD_INDEX_SIZE;
879+
__pgd_index_size = H_PGD_INDEX_SIZE;
880+
__pmd_cache_index = H_PMD_CACHE_INDEX;
881+
__pte_table_size = H_PTE_TABLE_SIZE;
882+
__pmd_table_size = H_PMD_TABLE_SIZE;
883+
__pud_table_size = H_PUD_TABLE_SIZE;
884+
__pgd_table_size = H_PGD_TABLE_SIZE;
873885
/* Initialize the MMU Hash table and create the linear mapping
874886
* of memory. Has to be done before SLB initialization as this is
875887
* currently where the page size encoding is obtained.

‎arch/powerpc/mm/init_64.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,11 @@
6666
#include "mmu_decl.h"
6767

6868
#ifdef CONFIG_PPC_STD_MMU_64
69-
#if PGTABLE_RANGE > USER_VSID_RANGE
69+
#if H_PGTABLE_RANGE > USER_VSID_RANGE
7070
#warning Limited user VSID range means pagetable space is wasted
7171
#endif
7272

73-
#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
73+
#if (TASK_SIZE_USER64 < H_PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
7474
#warning TASK_SIZE is smaller than it needs to be.
7575
#endif
7676
#endif /* CONFIG_PPC_STD_MMU_64 */

‎arch/powerpc/mm/pgtable-book3e.c

+1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ int map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flags)
7777
pmd_t *pmdp;
7878
pte_t *ptep;
7979

80+
BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE);
8081
if (slab_is_available()) {
8182
pgdp = pgd_offset_k(ea);
8283
pudp = pud_alloc(&init_mm, pgdp, ea);

‎arch/powerpc/mm/pgtable-hash64.c

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ int map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flags)
6161
pmd_t *pmdp;
6262
pte_t *ptep;
6363

64+
BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
6465
if (slab_is_available()) {
6566
pgdp = pgd_offset_k(ea);
6667
pudp = pud_alloc(&init_mm, pgdp, ea);

‎arch/powerpc/mm/pgtable_64.c

+22-11
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,6 @@
5858
#define CREATE_TRACE_POINTS
5959
#include <trace/events/thp.h>
6060

61-
/* Some sanity checking */
62-
#if TASK_SIZE_USER64 > PGTABLE_RANGE
63-
#error TASK_SIZE_USER64 exceeds pagetable range
64-
#endif
65-
6661
#ifdef CONFIG_PPC_STD_MMU_64
6762
#if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
6863
#error TASK_SIZE_USER64 exceeds user VSID range
@@ -75,6 +70,28 @@
7570
*/
7671
struct prtb_entry *process_tb;
7772
struct patb_entry *partition_tb;
73+
/*
74+
* page table size
75+
*/
76+
unsigned long __pte_index_size;
77+
EXPORT_SYMBOL(__pte_index_size);
78+
unsigned long __pmd_index_size;
79+
EXPORT_SYMBOL(__pmd_index_size);
80+
unsigned long __pud_index_size;
81+
EXPORT_SYMBOL(__pud_index_size);
82+
unsigned long __pgd_index_size;
83+
EXPORT_SYMBOL(__pgd_index_size);
84+
unsigned long __pmd_cache_index;
85+
EXPORT_SYMBOL(__pmd_cache_index);
86+
unsigned long __pte_table_size;
87+
EXPORT_SYMBOL(__pte_table_size);
88+
unsigned long __pmd_table_size;
89+
EXPORT_SYMBOL(__pmd_table_size);
90+
unsigned long __pud_table_size;
91+
EXPORT_SYMBOL(__pud_table_size);
92+
unsigned long __pgd_table_size;
93+
EXPORT_SYMBOL(__pgd_table_size);
94+
7895
#endif
7996
unsigned long ioremap_bot = IOREMAP_BASE;
8097

@@ -739,12 +756,6 @@ pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
739756
int has_transparent_hugepage(void)
740757
{
741758

742-
BUILD_BUG_ON_MSG((PMD_SHIFT - PAGE_SHIFT) >= MAX_ORDER,
743-
"hugepages can't be allocated by the buddy allocator");
744-
745-
BUILD_BUG_ON_MSG((PMD_SHIFT - PAGE_SHIFT) < 2,
746-
"We need more than 2 pages to do deferred thp split");
747-
748759
if (!mmu_has_feature(MMU_FTR_16M_PAGE))
749760
return 0;
750761
/*

‎arch/powerpc/mm/slb_low.S

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ _GLOBAL(slb_allocate_realmode)
3535
* check for bad kernel/user address
3636
* (ea & ~REGION_MASK) >= PGTABLE_RANGE
3737
*/
38-
rldicr. r9,r3,4,(63 - PGTABLE_EADDR_SIZE - 4)
38+
rldicr. r9,r3,4,(63 - H_PGTABLE_EADDR_SIZE - 4)
3939
bne- 8f
4040

4141
srdi r9,r3,60 /* get region */

‎arch/powerpc/mm/slice.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
#include <asm/hugetlb.h>
3838

3939
/* some sanity checks */
40-
#if (PGTABLE_RANGE >> 43) > SLICE_MASK_SIZE
41-
#error PGTABLE_RANGE exceeds slice_mask high_slices size
40+
#if (H_PGTABLE_RANGE >> 43) > SLICE_MASK_SIZE
41+
#error H_PGTABLE_RANGE exceeds slice_mask high_slices size
4242
#endif
4343

4444
static DEFINE_SPINLOCK(slice_convert_lock);

0 commit comments

Comments
 (0)