Skip to content

Commit

Permalink
mm, compaction: simplify contended compaction handling
Browse files Browse the repository at this point in the history 
Async compaction detects contention either due to failing trylock on
zone->lock or lru_lock, or by need_resched().  Since 1f9efde ("mm,
compaction: khugepaged should not give up due to need_resched()") the
code got quite complicated to distinguish these two up to the
__alloc_pages_slowpath() level, so different decisions could be taken
for khugepaged allocations.

After the recent changes, khugepaged allocations don't check for
contended compaction anymore, so we again don't need to distinguish lock
and sched contention, and simplify the current convoluted code a lot.

However, I believe it's also possible to simplify even more and
completely remove the check for contended compaction after the initial
async compaction for costly orders, which was originally aimed at THP
page fault allocations.  There are several reasons why this can be done
now:

- with the new defaults, THP page faults no longer do reclaim/compaction at
  all, unless the system admin has overridden the default, or application has
  indicated via madvise that it can benefit from THP's. In both cases, it
  means that the potential extra latency is expected and worth the benefits.
- even if reclaim/compaction proceeds after this patch where it previously
  wouldn't, the second compaction attempt is still async and will detect the
  contention and back off, if the contention persists
- there are still heuristics like deferred compaction and pageblock skip bits
  in place that prevent excessive THP page fault latencies

Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Vlastimil Babka <[email protected]>
Acked-by: Michal Hocko <[email protected]>
Acked-by: Mel Gorman <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@tehcaster @torvalds
tehcaster authored and torvalds committed Jul 28, 2016
1 parent a5508cd commit c3486f5
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 101 deletions.
13 changes: 2 additions & 11 deletions include/linux/compaction.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,6 @@ enum compact_result {
COMPACT_PARTIAL,
};

/* Used to signal whether compaction detected need_sched() or lock contention */
/* No contention detected */
#define COMPACT_CONTENDED_NONE 0
/* Either need_sched() was true or fatal signal pending */
#define COMPACT_CONTENDED_SCHED 1
/* Zone lock or lru_lock was contended in async compaction */
#define COMPACT_CONTENDED_LOCK 2

struct alloc_context; /* in mm/internal.h */

#ifdef CONFIG_COMPACTION
Expand All @@ -76,9 +68,8 @@ extern int sysctl_compact_unevictable_allowed;

extern int fragmentation_index(struct zone *zone, unsigned int order);
extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
unsigned int order,
unsigned int alloc_flags, const struct alloc_context *ac,
enum compact_priority prio, int *contended);
unsigned int order, unsigned int alloc_flags,
const struct alloc_context *ac, enum compact_priority prio);
extern void compact_pgdat(pg_data_t *pgdat, int order);
extern void reset_isolation_suitable(pg_data_t *pgdat);
extern enum compact_result compaction_suitable(struct zone *zone, int order,
Expand Down
72 changes: 13 additions & 59 deletions mm/compaction.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags,
{
if (cc->mode == MIGRATE_ASYNC) {
if (!spin_trylock_irqsave(lock, *flags)) {
cc->contended = COMPACT_CONTENDED_LOCK;
cc->contended = true;
return false;
}
} else {
Expand Down Expand Up @@ -365,13 +365,13 @@ static bool compact_unlock_should_abort(spinlock_t *lock,
}

if (fatal_signal_pending(current)) {
cc->contended = COMPACT_CONTENDED_SCHED;
cc->contended = true;
return true;
}

if (need_resched()) {
if (cc->mode == MIGRATE_ASYNC) {
cc->contended = COMPACT_CONTENDED_SCHED;
cc->contended = true;
return true;
}
cond_resched();
Expand All @@ -394,7 +394,7 @@ static inline bool compact_should_abort(struct compact_control *cc)
/* async compaction aborts if contended */
if (need_resched()) {
if (cc->mode == MIGRATE_ASYNC) {
cc->contended = COMPACT_CONTENDED_SCHED;
cc->contended = true;
return true;
}

Expand Down Expand Up @@ -1619,14 +1619,11 @@ static enum compact_result compact_zone(struct zone *zone, struct compact_contro
trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
cc->free_pfn, end_pfn, sync, ret);

if (ret == COMPACT_CONTENDED)
ret = COMPACT_PARTIAL;

return ret;
}

static enum compact_result compact_zone_order(struct zone *zone, int order,
gfp_t gfp_mask, enum compact_priority prio, int *contended,
gfp_t gfp_mask, enum compact_priority prio,
unsigned int alloc_flags, int classzone_idx)
{
enum compact_result ret;
Expand All @@ -1650,7 +1647,6 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
VM_BUG_ON(!list_empty(&cc.freepages));
VM_BUG_ON(!list_empty(&cc.migratepages));

*contended = cc.contended;
return ret;
}

Expand All @@ -1663,23 +1659,18 @@ int sysctl_extfrag_threshold = 500;
* @alloc_flags: The allocation flags of the current allocation
* @ac: The context of current allocation
* @mode: The migration mode for async, sync light, or sync migration
* @contended: Return value that determines if compaction was aborted due to
* need_resched() or lock contention
*
* This is the main entry point for direct page compaction.
*/
enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
unsigned int alloc_flags, const struct alloc_context *ac,
enum compact_priority prio, int *contended)
enum compact_priority prio)
{
int may_enter_fs = gfp_mask & __GFP_FS;
int may_perform_io = gfp_mask & __GFP_IO;
struct zoneref *z;
struct zone *zone;
enum compact_result rc = COMPACT_SKIPPED;
int all_zones_contended = COMPACT_CONTENDED_LOCK; /* init for &= op */

*contended = COMPACT_CONTENDED_NONE;

/* Check if the GFP flags allow compaction */
if (!may_enter_fs || !may_perform_io)
Expand All @@ -1691,22 +1682,15 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
ac->nodemask) {
enum compact_result status;
int zone_contended;

if (compaction_deferred(zone, order)) {
rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
continue;
}

status = compact_zone_order(zone, order, gfp_mask, prio,
&zone_contended, alloc_flags,
ac_classzone_idx(ac));
alloc_flags, ac_classzone_idx(ac));
rc = max(status, rc);
/*
* It takes at least one zone that wasn't lock contended
* to clear all_zones_contended.
*/
all_zones_contended &= zone_contended;

/* If a normal allocation would succeed, stop compacting */
if (zone_watermark_ok(zone, order, low_wmark_pages(zone),
Expand All @@ -1718,59 +1702,29 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
* succeeds in this zone.
*/
compaction_defer_reset(zone, order, false);
/*
* It is possible that async compaction aborted due to
* need_resched() and the watermarks were ok thanks to
* somebody else freeing memory. The allocation can
* however still fail so we better signal the
* need_resched() contention anyway (this will not
* prevent the allocation attempt).
*/
if (zone_contended == COMPACT_CONTENDED_SCHED)
*contended = COMPACT_CONTENDED_SCHED;

goto break_loop;
break;
}

if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
status == COMPACT_PARTIAL_SKIPPED)) {
status == COMPACT_PARTIAL_SKIPPED))
/*
* We think that allocation won't succeed in this zone
* so we defer compaction there. If it ends up
* succeeding after all, it will be reset.
*/
defer_compaction(zone, order);
}

/*
* We might have stopped compacting due to need_resched() in
* async compaction, or due to a fatal signal detected. In that
* case do not try further zones and signal need_resched()
* contention.
*/
if ((zone_contended == COMPACT_CONTENDED_SCHED)
|| fatal_signal_pending(current)) {
*contended = COMPACT_CONTENDED_SCHED;
goto break_loop;
}

continue;
break_loop:
/*
* We might not have tried all the zones, so be conservative
* and assume they are not all lock contended.
* case do not try further zones
*/
all_zones_contended = 0;
break;
if ((prio == COMPACT_PRIO_ASYNC && need_resched())
|| fatal_signal_pending(current))
break;
}

/*
* If at least one zone wasn't deferred or skipped, we report if all
* zones that were tried were lock contended.
*/
if (rc > COMPACT_INACTIVE && all_zones_contended)
*contended = COMPACT_CONTENDED_LOCK;

return rc;
}

Expand Down
5 changes: 1 addition & 4 deletions mm/internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,7 @@ struct compact_control {
const unsigned int alloc_flags; /* alloc flags of a direct compactor */
const int classzone_idx; /* zone index of a direct compactor */
struct zone *zone;
int contended; /* Signal need_sched() or lock
* contention detected during
* compaction
*/
bool contended; /* Signal lock or sched contention */
};

unsigned long
Expand Down
28 changes: 1 addition & 27 deletions mm/page_alloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3099,14 +3099,13 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
enum compact_priority prio, enum compact_result *compact_result)
{
struct page *page;
int contended_compaction;

if (!order)
return NULL;

current->flags |= PF_MEMALLOC;
*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
prio, &contended_compaction);
prio);
current->flags &= ~PF_MEMALLOC;

if (*compact_result <= COMPACT_INACTIVE)
Expand Down Expand Up @@ -3135,24 +3134,6 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
*/
count_vm_event(COMPACTFAIL);

/*
* In all zones where compaction was attempted (and not
* deferred or skipped), lock contention has been detected.
* For THP allocation we do not want to disrupt the others
* so we fallback to base pages instead.
*/
if (contended_compaction == COMPACT_CONTENDED_LOCK)
*compact_result = COMPACT_CONTENDED;

/*
* If compaction was aborted due to need_resched(), we do not
* want to further increase allocation latency, unless it is
* khugepaged trying to collapse.
*/
if (contended_compaction == COMPACT_CONTENDED_SCHED
&& !(current->flags & PF_KTHREAD))
*compact_result = COMPACT_CONTENDED;

cond_resched();

return NULL;
Expand Down Expand Up @@ -3547,13 +3528,6 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
if (compact_result == COMPACT_DEFERRED)
goto nopage;

/*
* Compaction is contended so rather back off than cause
* excessive stalls.
*/
if (compact_result == COMPACT_CONTENDED)
goto nopage;

/*
* Looks like reclaim/compaction is worth trying, but
* sync compaction could be very expensive, so keep
Expand Down

0 comments on commit c3486f5

Please sign in to comment.