diff --git a/include/linux/sched.h b/include/linux/sched.h
index ed3bb19d13372c..38c8654aaa96b6 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -785,7 +785,6 @@ struct task_struct {
 	short il_next;
 #endif
 #ifdef CONFIG_CPUSETS
-	short cpuset_sem_nest_depth;
 	struct cpuset *cpuset;
 	nodemask_t mems_allowed;
 	int cpuset_mems_generation;
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 407b5f0a8c8eee..79866bc6b3a154 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -180,6 +180,8 @@ static struct super_block *cpuset_sb = NULL;
  */
 
 static DECLARE_MUTEX(cpuset_sem);
+static struct task_struct *cpuset_sem_owner;
+static int cpuset_sem_depth;
 
 /*
  * The global cpuset semaphore cpuset_sem can be needed by the
@@ -200,16 +202,19 @@ static DECLARE_MUTEX(cpuset_sem);
 
 static inline void cpuset_down(struct semaphore *psem)
 {
-	if (current->cpuset_sem_nest_depth == 0)
+	if (cpuset_sem_owner != current) {
 		down(psem);
-	current->cpuset_sem_nest_depth++;
+		cpuset_sem_owner = current;
+	}
+	cpuset_sem_depth++;
 }
 
 static inline void cpuset_up(struct semaphore *psem)
 {
-	current->cpuset_sem_nest_depth--;
-	if (current->cpuset_sem_nest_depth == 0)
+	if (--cpuset_sem_depth == 0) {
+		cpuset_sem_owner = NULL;
 		up(psem);
+	}
 }
 
 /*