TT v0.3.2

jrglinux · Dec 10, 2021 · 42bdb9b · 42bdb9b
1 parent bf918c2
commit 42bdb9b
Showing 1 changed file with 37 additions and 15 deletions.
diff --git a/patches/5.15/tt-5.15.patch b/patches/5.15/tt-5.15.patch
@@ -2243,10 +2243,10 @@ index 000000000000..b3d99cf13576
 +#endif
 diff --git a/kernel/sched/bs_nohz.h b/kernel/sched/bs_nohz.h
 new file mode 100644
-index 000000000000..1bef67f7d871
+index 000000000000..f1ce0b979325
 --- /dev/null
 +++ b/kernel/sched/bs_nohz.h
-@@ -0,0 +1,869 @@
+@@ -0,0 +1,891 @@
 +
 +#ifdef CONFIG_NO_HZ_COMMON
 +
@@ -3004,7 +3004,7 @@ index 000000000000..1bef67f7d871
 +	UNLOCK_GRQ(grf);
 +}
 +
-+static void try_pull_from_grq(struct rq *dist_rq)
++static int try_pull_from_grq(struct rq *dist_rq)
 +{
 +	struct rq_flags rf;
 +	struct rq_flags grf;
@@ -3014,11 +3014,11 @@ index 000000000000..1bef67f7d871
 +	struct tt_node *ttn;
 +
 +	if (dist_rq == grq)
-+		return;
++		return 0;
 +
 +	/* if no tasks to pull, exit */
 +	if (!grq->cfs.head)
-+		return;
++		return 0;
 +
 +	rq_lock_irqsave(dist_rq, &rf);
 +	update_rq_clock(dist_rq);
@@ -3032,7 +3032,7 @@ index 000000000000..1bef67f7d871
 +	if (se_global == se_local) {
 +		rq_unlock(grq, &grf);
 +		local_irq_restore(grf.flags);
-+		return;
++		return 0;
 +	}
 +
 +	ttn = &se_global->tt_node;
@@ -3055,13 +3055,28 @@ index 000000000000..1bef67f7d871
 +	// unlock dist rq
 +	rq_unlock(dist_rq, &rf);
 +	local_irq_restore(grf.flags);
++	return 1;
++}
++
++static inline void
++update_grq_next_balance(struct rq *rq, int pulled)
++{
++	/*
++	 * if not pulled any, keep eager,
++	 * otherwise set next balance
++	 */
++	if (tt_grq_balance_ms && pulled)
++		rq->grq_next_balance = jiffies + msecs_to_jiffies(tt_grq_balance_ms);
 +}
 +
 +static void nohz_try_pull_from_grq(void)
 +{
 +	int cpu;
++	struct rq *rq;
 +	struct cpumask idle_mask;
 +	struct cpumask non_idle_mask;
++	bool balance_time;
++	int pulled = 0;
 +
 +	cpumask_clear(&non_idle_mask);
 +
@@ -3078,21 +3093,28 @@ index 000000000000..1bef67f7d871
 +
 +	/* second, idle cpus pull first */
 +	for_each_cpu(cpu, &idle_mask) {
-+		if (cpu == 0) continue;
-+		if (idle_cpu(cpu))
-+			pull_from_grq(cpu_rq(cpu));
++		if (cpu == 0 || !idle_cpu(cpu))
++			continue;
++
++		rq = cpu_rq(cpu);
++		pulled = pull_from_grq(rq);
++		update_grq_next_balance(rq, pulled);
 +	}
 +
 +	/* last, non idle pull */
 +	for_each_cpu(cpu, &non_idle_mask) {
++		rq = cpu_rq(cpu);
++		balance_time = time_after_eq(jiffies, rq->grq_next_balance);
++		pulled = 0;
++
 +		/* mybe it is idle now */
-+		if (idle_cpu(cpu)) {
-+			pull_from_grq(cpu_rq(cpu));
-+		} else if (time_after_eq(jiffies, grq->grq_next_balance)) {
++		if (idle_cpu(cpu))
++			pulled = pull_from_grq(cpu_rq(cpu));
++		else if (tt_grq_balance_ms == 0 || balance_time)
 +			/* if not idle, try pull every grq_next_balance */
-+			try_pull_from_grq(cpu_rq(cpu));
-+			grq->grq_next_balance = jiffies + msecs_to_jiffies(tt_grq_balance_ms);
-+		}
++			pulled = try_pull_from_grq(rq);
++
++		update_grq_next_balance(rq, pulled);
 +	}
 +}
 +