diff --git a/mm/rmap.c b/mm/rmap.c
index 450f5241b5a525..1fc559e09ca8c7 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -298,7 +298,11 @@ static int page_referenced_one(struct page *page,
 		if (ptep_clear_flush_young(vma, address, pte))
 			referenced++;
 
-		if (mm != current->mm && !ignore_token && has_swap_token(mm))
+		/* Pretend the page is referenced if the task has the
+		   swap token and is in the middle of a page fault. */
+		if (mm != current->mm && !ignore_token &&
+				has_swap_token(mm) &&
+				rwsem_is_locked(&mm->mmap_sem))
 			referenced++;
 
 		(*mapcount)--;
diff --git a/mm/thrash.c b/mm/thrash.c
index 11461f7ad8302c..eff3c18c33a10b 100644
--- a/mm/thrash.c
+++ b/mm/thrash.c
@@ -19,7 +19,7 @@ static unsigned long swap_token_check;
 struct mm_struct * swap_token_mm = &init_mm;
 
 #define SWAP_TOKEN_CHECK_INTERVAL (HZ * 2)
-#define SWAP_TOKEN_TIMEOUT	0
+#define SWAP_TOKEN_TIMEOUT	(300 * HZ)
 /*
  * Currently disabled; Needs further code to work at HZ * 300.
  */