Skip to content

Commit

Permalink
sched/deadline: Track the "total rq utilization" too
Browse files Browse the repository at this point in the history 
The total rq utilization is defined as the sum of the utilisations of
tasks that are "assigned" to a runqueue, independently from their state
(TASK_RUNNING or blocked)

Tested-by: Daniel Bristot de Oliveira <[email protected]>
Signed-off-by: Luca Abeni <[email protected]>
Signed-off-by: Claudio Scordino <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Cc: Joel Fernandes <[email protected]>
Cc: Juri Lelli <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Mathieu Poirier <[email protected]>
Cc: Mike Galbraith <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Steven Rostedt <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Tommaso Cucinotta <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
  • Loading branch information
Luca Abeni authored and Ingo Molnar committed Jun 8, 2017
1 parent 2d4283e commit 8fd2723
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 34 deletions.
118 changes: 84 additions & 34 deletions kernel/sched/deadline.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ void add_running_bw(u64 dl_bw, struct dl_rq *dl_rq)
lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);
dl_rq->running_bw += dl_bw;
SCHED_WARN_ON(dl_rq->running_bw < old); /* overflow */
SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);
}

static inline
Expand All @@ -65,25 +66,52 @@ void sub_running_bw(u64 dl_bw, struct dl_rq *dl_rq)
dl_rq->running_bw = 0;
}

static inline
void add_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)
{
u64 old = dl_rq->this_bw;

lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);
dl_rq->this_bw += dl_bw;
SCHED_WARN_ON(dl_rq->this_bw < old); /* overflow */
}

static inline
void sub_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)
{
u64 old = dl_rq->this_bw;

lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);
dl_rq->this_bw -= dl_bw;
SCHED_WARN_ON(dl_rq->this_bw > old); /* underflow */
if (dl_rq->this_bw > old)
dl_rq->this_bw = 0;
SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);
}

void dl_change_utilization(struct task_struct *p, u64 new_bw)
{
if (task_on_rq_queued(p))
return;
struct rq *rq;

if (!p->dl.dl_non_contending)
if (task_on_rq_queued(p))
return;

sub_running_bw(p->dl.dl_bw, &task_rq(p)->dl);
p->dl.dl_non_contending = 0;
/*
* If the timer handler is currently running and the
* timer cannot be cancelled, inactive_task_timer()
* will see that dl_not_contending is not set, and
* will not touch the rq's active utilization,
* so we are still safe.
*/
if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
put_task_struct(p);
rq = task_rq(p);
if (p->dl.dl_non_contending) {
sub_running_bw(p->dl.dl_bw, &rq->dl);
p->dl.dl_non_contending = 0;
/*
* If the timer handler is currently running and the
* timer cannot be cancelled, inactive_task_timer()
* will see that dl_not_contending is not set, and
* will not touch the rq's active utilization,
* so we are still safe.
*/
if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
put_task_struct(p);
}
sub_rq_bw(p->dl.dl_bw, &rq->dl);
add_rq_bw(new_bw, &rq->dl);
}

/*
Expand Down Expand Up @@ -178,6 +206,8 @@ static void task_non_contending(struct task_struct *p)
if (!dl_task(p) || p->state == TASK_DEAD) {
struct dl_bw *dl_b = dl_bw_of(task_cpu(p));

if (p->state == TASK_DEAD)
sub_rq_bw(p->dl.dl_bw, &rq->dl);
raw_spin_lock(&dl_b->lock);
__dl_clear(dl_b, p->dl.dl_bw);
__dl_clear_params(p);
Expand All @@ -192,7 +222,7 @@ static void task_non_contending(struct task_struct *p)
hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL);
}

static void task_contending(struct sched_dl_entity *dl_se)
static void task_contending(struct sched_dl_entity *dl_se, int flags)
{
struct dl_rq *dl_rq = dl_rq_of_se(dl_se);

Expand All @@ -203,6 +233,9 @@ static void task_contending(struct sched_dl_entity *dl_se)
if (dl_se->dl_runtime == 0)
return;

if (flags & ENQUEUE_MIGRATED)
add_rq_bw(dl_se->dl_bw, dl_rq);

if (dl_se->dl_non_contending) {
dl_se->dl_non_contending = 0;
/*
Expand Down Expand Up @@ -268,6 +301,7 @@ void init_dl_rq(struct dl_rq *dl_rq)
#endif

dl_rq->running_bw = 0;
dl_rq->this_bw = 0;
init_dl_rq_bw_ratio(dl_rq);
}

Expand Down Expand Up @@ -1042,6 +1076,7 @@ static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer)

if (p->state == TASK_DEAD && dl_se->dl_non_contending) {
sub_running_bw(p->dl.dl_bw, dl_rq_of_se(&p->dl));
sub_rq_bw(p->dl.dl_bw, dl_rq_of_se(&p->dl));
dl_se->dl_non_contending = 0;
}

Expand Down Expand Up @@ -1207,7 +1242,7 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se,
* we want a replenishment of its runtime.
*/
if (flags & ENQUEUE_WAKEUP) {
task_contending(dl_se);
task_contending(dl_se, flags);
update_dl_entity(dl_se, pi_se);
} else if (flags & ENQUEUE_REPLENISH) {
replenish_dl_entity(dl_se, pi_se);
Expand Down Expand Up @@ -1260,8 +1295,10 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
if (!p->dl.dl_throttled && dl_is_constrained(&p->dl))
dl_check_constrained_dl(&p->dl);

if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & ENQUEUE_RESTORE)
if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & ENQUEUE_RESTORE) {
add_rq_bw(p->dl.dl_bw, &rq->dl);
add_running_bw(p->dl.dl_bw, &rq->dl);
}

/*
* If p is throttled, we do not enqueue it. In fact, if it exhausted
Expand All @@ -1277,7 +1314,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
*/
if (p->dl.dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
if (flags & ENQUEUE_WAKEUP)
task_contending(&p->dl);
task_contending(&p->dl, flags);

return;
}
Expand All @@ -1299,8 +1336,10 @@ static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)
update_curr_dl(rq);
__dequeue_task_dl(rq, p, flags);

if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & DEQUEUE_SAVE)
if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & DEQUEUE_SAVE) {
sub_running_bw(p->dl.dl_bw, &rq->dl);
sub_rq_bw(p->dl.dl_bw, &rq->dl);
}

/*
* This check allows to start the inactive timer (or to immediately
Expand Down Expand Up @@ -1394,7 +1433,7 @@ static void migrate_task_rq_dl(struct task_struct *p)
{
struct rq *rq;

if (!(p->state == TASK_WAKING) || !(p->dl.dl_non_contending))
if (p->state != TASK_WAKING)
return;

rq = task_rq(p);
Expand All @@ -1404,18 +1443,20 @@ static void migrate_task_rq_dl(struct task_struct *p)
* rq->lock is not... So, lock it
*/
raw_spin_lock(&rq->lock);
sub_running_bw(p->dl.dl_bw, &rq->dl);
p->dl.dl_non_contending = 0;
/*
* If the timer handler is currently running and the
* timer cannot be cancelled, inactive_task_timer()
* will see that dl_not_contending is not set, and
* will not touch the rq's active utilization,
* so we are still safe.
*/
if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
put_task_struct(p);

if (p->dl.dl_non_contending) {
sub_running_bw(p->dl.dl_bw, &rq->dl);
p->dl.dl_non_contending = 0;
/*
* If the timer handler is currently running and the
* timer cannot be cancelled, inactive_task_timer()
* will see that dl_not_contending is not set, and
* will not touch the rq's active utilization,
* so we are still safe.
*/
if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
put_task_struct(p);
}
sub_rq_bw(p->dl.dl_bw, &rq->dl);
raw_spin_unlock(&rq->lock);
}

Expand Down Expand Up @@ -1858,7 +1899,9 @@ static int push_dl_task(struct rq *rq)

deactivate_task(rq, next_task, 0);
sub_running_bw(next_task->dl.dl_bw, &rq->dl);
sub_rq_bw(next_task->dl.dl_bw, &rq->dl);
set_task_cpu(next_task, later_rq->cpu);
add_rq_bw(next_task->dl.dl_bw, &later_rq->dl);
add_running_bw(next_task->dl.dl_bw, &later_rq->dl);
activate_task(later_rq, next_task, 0);
ret = 1;
Expand Down Expand Up @@ -1948,7 +1991,9 @@ static void pull_dl_task(struct rq *this_rq)

deactivate_task(src_rq, p, 0);
sub_running_bw(p->dl.dl_bw, &src_rq->dl);
sub_rq_bw(p->dl.dl_bw, &src_rq->dl);
set_task_cpu(p, this_cpu);
add_rq_bw(p->dl.dl_bw, &this_rq->dl);
add_running_bw(p->dl.dl_bw, &this_rq->dl);
activate_task(this_rq, p, 0);
dmin = p->dl.deadline;
Expand Down Expand Up @@ -2057,6 +2102,9 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p)
if (task_on_rq_queued(p) && p->dl.dl_runtime)
task_non_contending(p);

if (!task_on_rq_queued(p))
sub_rq_bw(p->dl.dl_bw, &rq->dl);

/*
* We cannot use inactive_task_timer() to invoke sub_running_bw()
* at the 0-lag time, because the task could have been migrated
Expand Down Expand Up @@ -2086,9 +2134,11 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)
put_task_struct(p);

/* If p is not queued we will update its parameters at next wakeup. */
if (!task_on_rq_queued(p))
return;
if (!task_on_rq_queued(p)) {
add_rq_bw(p->dl.dl_bw, &rq->dl);

return;
}
/*
* If p is boosted we already updated its params in
* rt_mutex_setprio()->enqueue_task(..., ENQUEUE_REPLENISH),
Expand Down
11 changes: 11 additions & 0 deletions kernel/sched/sched.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,17 @@ struct dl_rq {
*/
u64 running_bw;

/*
* Utilization of the tasks "assigned" to this runqueue (including
* the tasks that are in runqueue and the tasks that executed on this
* CPU and blocked). Increased when a task moves to this runqueue, and
* decreased when the task moves away (migrates, changes scheduling
* policy, or terminates).
* This is needed to compute the "inactive utilization" for the
* runqueue (inactive utilization = this_bw - running_bw).
*/
u64 this_bw;

/*
* Inverse of the fraction of CPU utilization that can be reclaimed
* by the GRUB algorithm.
Expand Down

0 comments on commit 8fd2723

Please sign in to comment.