JakeHillion/scx-upstream
1
0
Fork 0
mirror of https://github.com/sched-ext/scx.git synced 2024-11-24 20:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

scx_bpfland: improve turbo domain CPU selection

Browse Source 
Always consider the turbo domain when running in "auto" mode.

Additionally, when the turbo domain is used, split the CPU idle
selection logic into two stages:
 1) in ops.select_cpu(), provide the task with a second opportunity to
    remain within the same LLC
 2) in ops.enqueue(), perform another check for an idle CPU, allowing
    the task to move to a different LLC if an idle CPU within the same
    LLC is not available.

This allows tasks to stick more on turbo-boosted CPUs and CPUs within
the same LLC.

Signed-off-by: Andrea Righi <andrea.righi@linux.dev>
This commit is contained in:
Andrea Righi 2024-09-01 16:18:02 +02:00
parent 70b93ed641
commit fe6ac15015
2 changed files with 24 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
scheds/rust/scx_bpfland/src/bpf/main.bpf.c
View File

@ -501,14 +501,13 @@ static int dispatch_direct_cpu(struct task_struct *p, s32 cpu, u64 enq_flags)
* to handle these mistakes in favor of a more efficient response and a reduced
* scheduling overhead.
*/
static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu)
static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo)
{
const struct cpumask *online_cpumask, *idle_smtmask, *idle_cpumask;
struct bpf_cpumask *primary, *turbo, *l2_domain, *l3_domain;
struct bpf_cpumask *p_mask, *l2_mask, *l3_mask;
struct task_ctx *tctx;
struct cpu_ctx *cctx;
bool do_turbo = true;
s32 cpu;
tctx = try_lookup_task_ctx(p);
@ -566,7 +565,7 @@ static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu)
l3_domain = cctx->l3_cpumask;
if (!l3_domain)
l3_domain = primary;
retry:
/*
* Task's scheduling domains.
*/
@ -663,6 +662,15 @@ retry:
goto out_put_cpumask;
}
/*
* When considering the turbo domain (first idle CPU selection
* pass) try to stay on the same LLC.
*/
if (do_turbo) {
cpu = -ENOENT;
goto out_put_cpumask;
}
/*
* Search for any other full-idle core in the primary domain.
*/
@ -704,6 +712,15 @@ retry:
goto out_put_cpumask;
}
/*
* When considering the turbo domain (first idle CPU selection pass)
* try to stay on the same LLC.
*/
if (do_turbo) {
cpu = -ENOENT;
goto out_put_cpumask;
}
/*
* Search for any idle CPU in the scheduling domain.
*/
@ -712,15 +729,6 @@ retry:
scx_bpf_test_and_clear_cpu_idle(cpu))
goto out_put_cpumask;
/*
* If we were looking for an idle CPU in the turbo domain and we
* couldn't find any, re-try again with the whole primary domain.
*/
if (do_turbo) {
do_turbo = false;
goto retry;
}
/*
* We couldn't find any idle CPU, so simply dispatch the task to the
* first CPU that will become available.
@ -747,7 +755,7 @@ s32 BPF_STRUCT_OPS(bpfland_select_cpu, struct task_struct *p, s32 prev_cpu, u64
{
s32 cpu;
cpu = pick_idle_cpu(p, prev_cpu);
cpu = pick_idle_cpu(p, prev_cpu, true);
if (cpu >= 0 && !dispatch_direct_cpu(p, cpu, 0)) {
__sync_fetch_and_add(&nr_direct_dispatches, 1);
return cpu;
@ -804,7 +812,7 @@ void BPF_STRUCT_OPS(bpfland_enqueue, struct task_struct *p, u64 enq_flags)
* If we couldn't find an idle CPU in ops.select_cpu(), give the task
* another chance here to keep using the same CPU / cache / domain.
*/
cpu = pick_idle_cpu(p, prev_cpu);
cpu = pick_idle_cpu(p, prev_cpu, false);
if (cpu >= 0 && !dispatch_direct_cpu(p, cpu, 0)) {
__sync_fetch_and_add(&nr_direct_dispatches, 1);
return;

22
scheds/rust/scx_bpfland/src/main.rs
View File

@ -286,7 +286,7 @@ impl<'a> Scheduler<'a> {
// Initialize the primary scheduling domain (based on the --primary-domain option).
let energy_profile = Self::read_energy_profile();
if let Err(err) = Self::init_turbo_domain(&mut skel, &opts.primary_domain, &energy_profile)
if let Err(err) = Self::init_turbo_domain(&mut skel, &opts.primary_domain)
{
warn!("failed to initialize turbo domain: error {}", err);
}
@ -404,27 +404,10 @@ impl<'a> Scheduler<'a> {
fn init_turbo_domain(
skel: &mut BpfSkel<'_>,
primary_domain: &String,
energy_profile: &String,
) -> Result<()> {
// Scheduling only on the turbo boosted CPUs can actually help to save power, because we
// end up using less CPUs, for a shorter amount of time, and they can go idle quickly, so
// use the turbo boosted domain only when we are trying to save power.
let domain = match primary_domain.as_str() {
"powersave" => {
Self::epp_to_cpumask(Powermode::Turbo)?
}
"performance" => {
Cpumask::new()?
}
"auto" => {
match energy_profile.as_str() {
"balance_power" | "power" => {
Self::epp_to_cpumask(Powermode::Turbo)?
}
&_ => {
Cpumask::new()?
}
}
Self::epp_to_cpumask(Powermode::Turbo)?
}
&_ => {
Cpumask::new()?
@ -538,7 +521,6 @@ impl<'a> Scheduler<'a> {
if let Err(err) = Self::init_turbo_domain(
&mut self.skel,
&self.opts.primary_domain,
&energy_profile,
) {
warn!("failed to refresh turbo domain: error {}", err);
}