Skip to content

Commit

Permalink
mm/sparsemem: convert kmalloc_section_memmap() to populate_section_me…
Browse files Browse the repository at this point in the history 
…mmap()

Allow sub-section sized ranges to be added to the memmap.

populate_section_memmap() takes an explict pfn range rather than
assuming a full section, and those parameters are plumbed all the way
through to vmmemap_populate().  There should be no sub-section usage in
current deployments.  New warnings are added to clarify which memmap
allocation paths are sub-section capable.

Link: http://lkml.kernel.org/r/156092352058.979959.6551283472062305149.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <[email protected]>
Reviewed-by: Pavel Tatashin <[email protected]>
Tested-by: Aneesh Kumar K.V <[email protected]>	[ppc64]
Reviewed-by: Oscar Salvador <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: David Hildenbrand <[email protected]>
Cc: Logan Gunthorpe <[email protected]>
Cc: Jane Chu <[email protected]>
Cc: Jeff Moyer <[email protected]>
Cc: Jérôme Glisse <[email protected]>
Cc: Jonathan Corbet <[email protected]>
Cc: Mike Rapoport <[email protected]>
Cc: Toshi Kani <[email protected]>
Cc: Vlastimil Babka <[email protected]>
Cc: Wei Yang <[email protected]>
Cc: Jason Gunthorpe <[email protected]>
Cc: Christoph Hellwig <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@djbw @torvalds
djbw authored and torvalds committed Jul 19, 2019
1 parent 49ba3c6 commit e9c0a3f
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 33 deletions.
4 changes: 3 additions & 1 deletion arch/x86/mm/init_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1518,7 +1518,9 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
{
int err;

if (boot_cpu_has(X86_FEATURE_PSE))
if (end - start < PAGES_PER_SECTION * sizeof(struct page))
err = vmemmap_populate_basepages(start, end, node);
else if (boot_cpu_has(X86_FEATURE_PSE))
err = vmemmap_populate_hugepages(start, end, node, altmap);
else if (altmap) {
pr_err_once("%s: no cpu support for altmap allocations\n",
Expand Down
4 changes: 2 additions & 2 deletions include/linux/mm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2767,8 +2767,8 @@ static inline void print_vma_addr(char *prefix, unsigned long rip)
#endif

void *sparse_buffer_alloc(unsigned long size);
struct page *sparse_mem_map_populate(unsigned long pnum, int nid,
struct vmem_altmap *altmap);
struct page * __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap);
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
Expand Down
21 changes: 14 additions & 7 deletions mm/sparse-vmemmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,19 +245,26 @@ int __meminit vmemmap_populate_basepages(unsigned long start,
return 0;
}

struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid,
struct vmem_altmap *altmap)
struct page * __meminit __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
{
unsigned long start;
unsigned long end;
struct page *map;

map = pfn_to_page(pnum * PAGES_PER_SECTION);
start = (unsigned long)map;
end = (unsigned long)(map + PAGES_PER_SECTION);
/*
* The minimum granularity of memmap extensions is
* PAGES_PER_SUBSECTION as allocations are tracked in the
* 'subsection_map' bitmap of the section.
*/
end = ALIGN(pfn + nr_pages, PAGES_PER_SUBSECTION);
pfn &= PAGE_SUBSECTION_MASK;
nr_pages = end - pfn;

start = (unsigned long) pfn_to_page(pfn);
end = start + nr_pages * sizeof(struct page);

if (vmemmap_populate(start, end, nid, altmap))
return NULL;

return map;
return pfn_to_page(pfn);
}
50 changes: 27 additions & 23 deletions mm/sparse.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,8 +439,8 @@ static unsigned long __init section_map_size(void)
return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
}

struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid,
struct vmem_altmap *altmap)
struct page __init *__populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
{
unsigned long size = section_map_size();
struct page *map = sparse_buffer_alloc(size);
Expand Down Expand Up @@ -521,10 +521,13 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
}
sparse_buffer_init(map_count * section_map_size(), nid);
for_each_present_section_nr(pnum_begin, pnum) {
unsigned long pfn = section_nr_to_pfn(pnum);

if (pnum >= pnum_end)
break;

map = sparse_mem_map_populate(pnum, nid, NULL);
map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
nid, NULL);
if (!map) {
pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
__func__, nid);
Expand Down Expand Up @@ -625,17 +628,17 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
#endif

#ifdef CONFIG_SPARSEMEM_VMEMMAP
static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
struct vmem_altmap *altmap)
static struct page *populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
{
/* This will make the necessary allocations eventually. */
return sparse_mem_map_populate(pnum, nid, altmap);
return __populate_section_memmap(pfn, nr_pages, nid, altmap);
}
static void __kfree_section_memmap(struct page *memmap,

static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap)
{
unsigned long start = (unsigned long)memmap;
unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page);

vmemmap_free(start, end, altmap);
}
Expand All @@ -647,7 +650,8 @@ static void free_map_bootmem(struct page *memmap)
vmemmap_free(start, end, NULL);
}
#else
static struct page *__kmalloc_section_memmap(void)
struct page *populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
{
struct page *page, *ret;
unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION;
Expand All @@ -668,15 +672,11 @@ static struct page *__kmalloc_section_memmap(void)
return ret;
}

static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap)
{
return __kmalloc_section_memmap();
}
struct page *memmap = pfn_to_page(pfn);

static void __kfree_section_memmap(struct page *memmap,
struct vmem_altmap *altmap)
{
if (is_vmalloc_addr(memmap))
vfree(memmap);
else
Expand Down Expand Up @@ -745,12 +745,13 @@ int __meminit sparse_add_one_section(int nid, unsigned long start_pfn,
if (ret < 0 && ret != -EEXIST)
return ret;
ret = 0;
memmap = kmalloc_section_memmap(section_nr, nid, altmap);
memmap = populate_section_memmap(start_pfn, PAGES_PER_SECTION, nid,
altmap);
if (!memmap)
return -ENOMEM;
usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
if (!usage) {
__kfree_section_memmap(memmap, altmap);
depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap);
return -ENOMEM;
}

Expand All @@ -773,7 +774,7 @@ int __meminit sparse_add_one_section(int nid, unsigned long start_pfn,
out:
if (ret < 0) {
kfree(usage);
__kfree_section_memmap(memmap, altmap);
depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap);
}
return ret;
}
Expand Down Expand Up @@ -809,7 +810,8 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
#endif

static void free_section_usage(struct mem_section *ms, struct page *memmap,
struct mem_section_usage *usage, struct vmem_altmap *altmap)
struct mem_section_usage *usage, unsigned long pfn,
unsigned long nr_pages, struct vmem_altmap *altmap)
{
if (!usage)
return;
Expand All @@ -820,7 +822,7 @@ static void free_section_usage(struct mem_section *ms, struct page *memmap,
if (!early_section(ms)) {
kfree(usage);
if (memmap)
__kfree_section_memmap(memmap, altmap);
depopulate_section_memmap(pfn, nr_pages, altmap);
return;
}

Expand Down Expand Up @@ -849,6 +851,8 @@ void sparse_remove_one_section(struct mem_section *ms, unsigned long map_offset,

clear_hwpoisoned_pages(memmap + map_offset,
PAGES_PER_SECTION - map_offset);
free_section_usage(ms, memmap, usage, altmap);
free_section_usage(ms, memmap, usage,
section_nr_to_pfn(__section_nr(ms)),
PAGES_PER_SECTION, altmap);
}
#endif /* CONFIG_MEMORY_HOTPLUG */

0 comments on commit e9c0a3f

Please sign in to comment.