Skip to content

Commit 9057289

Browse files
Peter ZijlstraIngo Molnar
Peter Zijlstra
authored and
Ingo Molnar
committed
mm: numa: Change page last {nid,pid} into {cpu,pid}
Change the per page last fault tracking to use cpu,pid instead of nid,pid. This will allow us to try and lookup the alternate task more easily. Note that even though it is the cpu that is store in the page flags that the mpol_misplaced decision is still based on the node. Signed-off-by: Peter Zijlstra <[email protected]> Signed-off-by: Mel Gorman <[email protected]> Reviewed-by: Rik van Riel <[email protected]> Cc: Andrea Arcangeli <[email protected]> Cc: Johannes Weiner <[email protected]> Cc: Srikar Dronamraju <[email protected]> Link: http://lkml.kernel.org/r/[email protected] [ Fixed build failure on 32-bit systems. ] Signed-off-by: Ingo Molnar <[email protected]>
1 parent e1dda8a commit 9057289

13 files changed

+125
-109
lines changed

‎include/linux/mm.h

+50-40
Original file line numberDiff line numberDiff line change
@@ -581,11 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
581581
* sets it, so none of the operations on it need to be atomic.
582582
*/
583583

584-
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NIDPID] | ... | FLAGS | */
584+
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
585585
#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
586586
#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
587587
#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
588-
#define LAST_NIDPID_PGOFF (ZONES_PGOFF - LAST_NIDPID_WIDTH)
588+
#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
589589

590590
/*
591591
* Define the bit shifts to access each section. For non-existent
@@ -595,7 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
595595
#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
596596
#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
597597
#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
598-
#define LAST_NIDPID_PGSHIFT (LAST_NIDPID_PGOFF * (LAST_NIDPID_WIDTH != 0))
598+
#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
599599

600600
/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
601601
#ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -617,7 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
617617
#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
618618
#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
619619
#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
620-
#define LAST_NIDPID_MASK ((1UL << LAST_NIDPID_WIDTH) - 1)
620+
#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_WIDTH) - 1)
621621
#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
622622

623623
static inline enum zone_type page_zonenum(const struct page *page)
@@ -661,96 +661,106 @@ static inline int page_to_nid(const struct page *page)
661661
#endif
662662

663663
#ifdef CONFIG_NUMA_BALANCING
664-
static inline int nid_pid_to_nidpid(int nid, int pid)
664+
static inline int cpu_pid_to_cpupid(int cpu, int pid)
665665
{
666-
return ((nid & LAST__NID_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
666+
return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
667667
}
668668

669-
static inline int nidpid_to_pid(int nidpid)
669+
static inline int cpupid_to_pid(int cpupid)
670670
{
671-
return nidpid & LAST__PID_MASK;
671+
return cpupid & LAST__PID_MASK;
672672
}
673673

674-
static inline int nidpid_to_nid(int nidpid)
674+
static inline int cpupid_to_cpu(int cpupid)
675675
{
676-
return (nidpid >> LAST__PID_SHIFT) & LAST__NID_MASK;
676+
return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
677677
}
678678

679-
static inline bool nidpid_pid_unset(int nidpid)
679+
static inline int cpupid_to_nid(int cpupid)
680680
{
681-
return nidpid_to_pid(nidpid) == (-1 & LAST__PID_MASK);
681+
return cpu_to_node(cpupid_to_cpu(cpupid));
682682
}
683683

684-
static inline bool nidpid_nid_unset(int nidpid)
684+
static inline bool cpupid_pid_unset(int cpupid)
685685
{
686-
return nidpid_to_nid(nidpid) == (-1 & LAST__NID_MASK);
686+
return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
687687
}
688688

689-
#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
690-
static inline int page_nidpid_xchg_last(struct page *page, int nid)
689+
static inline bool cpupid_cpu_unset(int cpupid)
691690
{
692-
return xchg(&page->_last_nidpid, nid);
691+
return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
693692
}
694693

695-
static inline int page_nidpid_last(struct page *page)
694+
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
695+
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
696696
{
697-
return page->_last_nidpid;
697+
return xchg(&page->_last_cpupid, cpupid);
698698
}
699-
static inline void page_nidpid_reset_last(struct page *page)
699+
700+
static inline int page_cpupid_last(struct page *page)
701+
{
702+
return page->_last_cpupid;
703+
}
704+
static inline void page_cpupid_reset_last(struct page *page)
700705
{
701-
page->_last_nidpid = -1;
706+
page->_last_cpupid = -1;
702707
}
703708
#else
704-
static inline int page_nidpid_last(struct page *page)
709+
static inline int page_cpupid_last(struct page *page)
705710
{
706-
return (page->flags >> LAST_NIDPID_PGSHIFT) & LAST_NIDPID_MASK;
711+
return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
707712
}
708713

709-
extern int page_nidpid_xchg_last(struct page *page, int nidpid);
714+
extern int page_cpupid_xchg_last(struct page *page, int cpupid);
710715

711-
static inline void page_nidpid_reset_last(struct page *page)
716+
static inline void page_cpupid_reset_last(struct page *page)
712717
{
713-
int nidpid = (1 << LAST_NIDPID_SHIFT) - 1;
718+
int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
714719

715-
page->flags &= ~(LAST_NIDPID_MASK << LAST_NIDPID_PGSHIFT);
716-
page->flags |= (nidpid & LAST_NIDPID_MASK) << LAST_NIDPID_PGSHIFT;
720+
page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
721+
page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
717722
}
718-
#endif /* LAST_NIDPID_NOT_IN_PAGE_FLAGS */
719-
#else
720-
static inline int page_nidpid_xchg_last(struct page *page, int nidpid)
723+
#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
724+
#else /* !CONFIG_NUMA_BALANCING */
725+
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
721726
{
722-
return page_to_nid(page);
727+
return page_to_nid(page); /* XXX */
723728
}
724729

725-
static inline int page_nidpid_last(struct page *page)
730+
static inline int page_cpupid_last(struct page *page)
726731
{
727-
return page_to_nid(page);
732+
return page_to_nid(page); /* XXX */
728733
}
729734

730-
static inline int nidpid_to_nid(int nidpid)
735+
static inline int cpupid_to_nid(int cpupid)
731736
{
732737
return -1;
733738
}
734739

735-
static inline int nidpid_to_pid(int nidpid)
740+
static inline int cpupid_to_pid(int cpupid)
736741
{
737742
return -1;
738743
}
739744

740-
static inline int nid_pid_to_nidpid(int nid, int pid)
745+
static inline int cpupid_to_cpu(int cpupid)
741746
{
742747
return -1;
743748
}
744749

745-
static inline bool nidpid_pid_unset(int nidpid)
750+
static inline int cpu_pid_to_cpupid(int nid, int pid)
751+
{
752+
return -1;
753+
}
754+
755+
static inline bool cpupid_pid_unset(int cpupid)
746756
{
747757
return 1;
748758
}
749759

750-
static inline void page_nidpid_reset_last(struct page *page)
760+
static inline void page_cpupid_reset_last(struct page *page)
751761
{
752762
}
753-
#endif
763+
#endif /* CONFIG_NUMA_BALANCING */
754764

755765
static inline struct zone *page_zone(const struct page *page)
756766
{

‎include/linux/mm_types.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,8 @@ struct page {
174174
void *shadow;
175175
#endif
176176

177-
#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
178-
int _last_nidpid;
177+
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
178+
int _last_cpupid;
179179
#endif
180180
}
181181
/*

‎include/linux/page-flags-layout.h

+11-11
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@
3939
* lookup is necessary.
4040
*
4141
* No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
42-
* " plus space for last_nidpid: | NODE | ZONE | LAST_NIDPID ... | FLAGS |
42+
* " plus space for last_cpupid: | NODE | ZONE | LAST_CPUPID ... | FLAGS |
4343
* classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
44-
* " plus space for last_nidpid: | SECTION | NODE | ZONE | LAST_NIDPID ... | FLAGS |
44+
* " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |
4545
* classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
4646
*/
4747
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
@@ -65,18 +65,18 @@
6565
#define LAST__PID_SHIFT 8
6666
#define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1)
6767

68-
#define LAST__NID_SHIFT NODES_SHIFT
69-
#define LAST__NID_MASK ((1 << LAST__NID_SHIFT)-1)
68+
#define LAST__CPU_SHIFT NR_CPUS_BITS
69+
#define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1)
7070

71-
#define LAST_NIDPID_SHIFT (LAST__PID_SHIFT+LAST__NID_SHIFT)
71+
#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
7272
#else
73-
#define LAST_NIDPID_SHIFT 0
73+
#define LAST_CPUPID_SHIFT 0
7474
#endif
7575

76-
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_NIDPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
77-
#define LAST_NIDPID_WIDTH LAST_NIDPID_SHIFT
76+
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
77+
#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
7878
#else
79-
#define LAST_NIDPID_WIDTH 0
79+
#define LAST_CPUPID_WIDTH 0
8080
#endif
8181

8282
/*
@@ -87,8 +87,8 @@
8787
#define NODE_NOT_IN_PAGE_FLAGS
8888
#endif
8989

90-
#if defined(CONFIG_NUMA_BALANCING) && LAST_NIDPID_WIDTH == 0
91-
#define LAST_NIDPID_NOT_IN_PAGE_FLAGS
90+
#if defined(CONFIG_NUMA_BALANCING) && LAST_CPUPID_WIDTH == 0
91+
#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
9292
#endif
9393

9494
#endif /* _LINUX_PAGE_FLAGS_LAYOUT */

‎kernel/bounds.c

+4
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@
1010
#include <linux/mmzone.h>
1111
#include <linux/kbuild.h>
1212
#include <linux/page_cgroup.h>
13+
#include <linux/log2.h>
1314

1415
void foo(void)
1516
{
1617
/* The enum constants to put into include/generated/bounds.h */
1718
DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);
1819
DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES);
1920
DEFINE(NR_PCG_FLAGS, __NR_PCG_FLAGS);
21+
#ifdef CONFIG_SMP
22+
DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
23+
#endif
2024
/* End of constants */
2125
}

‎kernel/sched/fair.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -1210,7 +1210,7 @@ static void task_numa_placement(struct task_struct *p)
12101210
/*
12111211
* Got a PROT_NONE fault for a page on @node.
12121212
*/
1213-
void task_numa_fault(int last_nidpid, int node, int pages, bool migrated)
1213+
void task_numa_fault(int last_cpupid, int node, int pages, bool migrated)
12141214
{
12151215
struct task_struct *p = current;
12161216
int priv;
@@ -1226,8 +1226,8 @@ void task_numa_fault(int last_nidpid, int node, int pages, bool migrated)
12261226
* First accesses are treated as private, otherwise consider accesses
12271227
* to be private if the accessing pid has not changed
12281228
*/
1229-
if (!nidpid_pid_unset(last_nidpid))
1230-
priv = ((p->pid & LAST__PID_MASK) == nidpid_to_pid(last_nidpid));
1229+
if (!cpupid_pid_unset(last_cpupid))
1230+
priv = ((p->pid & LAST__PID_MASK) == cpupid_to_pid(last_cpupid));
12311231
else
12321232
priv = 1;
12331233

‎mm/huge_memory.c

+4-4
Original file line numberDiff line numberDiff line change
@@ -1282,7 +1282,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
12821282
struct page *page;
12831283
unsigned long haddr = addr & HPAGE_PMD_MASK;
12841284
int page_nid = -1, this_nid = numa_node_id();
1285-
int target_nid, last_nidpid = -1;
1285+
int target_nid, last_cpupid = -1;
12861286
bool page_locked;
12871287
bool migrated = false;
12881288

@@ -1293,7 +1293,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
12931293
page = pmd_page(pmd);
12941294
BUG_ON(is_huge_zero_page(page));
12951295
page_nid = page_to_nid(page);
1296-
last_nidpid = page_nidpid_last(page);
1296+
last_cpupid = page_cpupid_last(page);
12971297
count_vm_numa_event(NUMA_HINT_FAULTS);
12981298
if (page_nid == this_nid)
12991299
count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
@@ -1362,7 +1362,7 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
13621362
page_unlock_anon_vma_read(anon_vma);
13631363

13641364
if (page_nid != -1)
1365-
task_numa_fault(last_nidpid, page_nid, HPAGE_PMD_NR, migrated);
1365+
task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, migrated);
13661366

13671367
return 0;
13681368
}
@@ -1682,7 +1682,7 @@ static void __split_huge_page_refcount(struct page *page,
16821682
page_tail->mapping = page->mapping;
16831683

16841684
page_tail->index = page->index + i;
1685-
page_nidpid_xchg_last(page_tail, page_nidpid_last(page));
1685+
page_cpupid_xchg_last(page_tail, page_cpupid_last(page));
16861686

16871687
BUG_ON(!PageAnon(page_tail));
16881688
BUG_ON(!PageUptodate(page_tail));

‎mm/memory.c

+8-8
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@
6969

7070
#include "internal.h"
7171

72-
#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
73-
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_nidpid.
72+
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
73+
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
7474
#endif
7575

7676
#ifndef CONFIG_NEED_MULTIPLE_NODES
@@ -3536,7 +3536,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
35363536
struct page *page = NULL;
35373537
spinlock_t *ptl;
35383538
int page_nid = -1;
3539-
int last_nidpid;
3539+
int last_cpupid;
35403540
int target_nid;
35413541
bool migrated = false;
35423542

@@ -3567,7 +3567,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
35673567
}
35683568
BUG_ON(is_zero_pfn(page_to_pfn(page)));
35693569

3570-
last_nidpid = page_nidpid_last(page);
3570+
last_cpupid = page_cpupid_last(page);
35713571
page_nid = page_to_nid(page);
35723572
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
35733573
pte_unmap_unlock(ptep, ptl);
@@ -3583,7 +3583,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
35833583

35843584
out:
35853585
if (page_nid != -1)
3586-
task_numa_fault(last_nidpid, page_nid, 1, migrated);
3586+
task_numa_fault(last_cpupid, page_nid, 1, migrated);
35873587
return 0;
35883588
}
35893589

@@ -3598,7 +3598,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
35983598
unsigned long offset;
35993599
spinlock_t *ptl;
36003600
bool numa = false;
3601-
int last_nidpid;
3601+
int last_cpupid;
36023602

36033603
spin_lock(&mm->page_table_lock);
36043604
pmd = *pmdp;
@@ -3643,7 +3643,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
36433643
if (unlikely(!page))
36443644
continue;
36453645

3646-
last_nidpid = page_nidpid_last(page);
3646+
last_cpupid = page_cpupid_last(page);
36473647
page_nid = page_to_nid(page);
36483648
target_nid = numa_migrate_prep(page, vma, addr, page_nid);
36493649
pte_unmap_unlock(pte, ptl);
@@ -3656,7 +3656,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
36563656
}
36573657

36583658
if (page_nid != -1)
3659-
task_numa_fault(last_nidpid, page_nid, 1, migrated);
3659+
task_numa_fault(last_cpupid, page_nid, 1, migrated);
36603660

36613661
pte = pte_offset_map_lock(mm, pmdp, addr, &ptl);
36623662
}

0 commit comments

Comments
 (0)