JakeHillion/scx
1
0
Fork 0
mirror of https://github.com/JakeHillion/scx.git synced 2024-11-26 03:20:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

scx_bpfland: enable "auto" mode by default

Browse Source 
Rename "turbo domain" to "preferred domain", that conceptually is more
generic and introduce the new option `--preferred-domain CPUMASK`, which
allows users to define the preferred domain, specifying a cpumask as a
hex number. By default ("auto") the scheduler will always try to detect
and use the fastest CPUs in the system.

Moreover, adjust the cpufreq logic to use "auto" both with the
"balance_power" and "balance_performance" EPP profiles.

Then, enable "auto" mode by default: the scheduler will try to
automatically determine the optimal primary domain, preferred domain and
cpufreq level, based on the selected scheduler and energy profiles.

Tested-by: Piotr Gorski < piotr.gorski@cachyos.org >
Signed-off-by: Andrea Righi <andrea.righi@linux.dev>
This commit is contained in:
Andrea Righi 2024-09-05 13:09:52 +02:00
parent 239b5194a4
commit 844c00fd26
2 changed files with 61 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
scheds/rust/scx_bpfland/src/bpf/main.bpf.c
View File

@ -140,9 +140,9 @@ UEI_DEFINE(uei);
private(BPFLAND) struct bpf_cpumask __kptr *primary_cpumask; private(BPFLAND) struct bpf_cpumask __kptr *primary_cpumask;
/* /*
* Mask of turbo boosted CPUs in the system. * Mask of preferred CPUs in the system.
*/ */
private(BPFLAND) struct bpf_cpumask __kptr *turbo_cpumask; private(BPFLAND) struct bpf_cpumask __kptr *preferred_cpumask;
/* /*
* Mask of offline CPUs, used to properly support CPU hotplugging. * Mask of offline CPUs, used to properly support CPU hotplugging.
@ -504,10 +504,10 @@ static int dispatch_direct_cpu(struct task_struct *p, s32 cpu, u64 enq_flags)
* to handle these mistakes in favor of a more efficient response and a reduced * to handle these mistakes in favor of a more efficient response and a reduced
* scheduling overhead. * scheduling overhead.
*/ */
static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo) static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_preferred)
{ {
const struct cpumask *online_cpumask, *idle_smtmask, *idle_cpumask; const struct cpumask *online_cpumask, *idle_smtmask, *idle_cpumask;
struct bpf_cpumask *primary, *turbo, *l2_domain, *l3_domain; struct bpf_cpumask *primary, *preferred, *l2_domain, *l3_domain;
struct bpf_cpumask *p_mask, *l2_mask, *l3_mask; struct bpf_cpumask *p_mask, *l2_mask, *l3_mask;
struct task_ctx *tctx; struct task_ctx *tctx;
struct cpu_ctx *cctx; struct cpu_ctx *cctx;
@ -523,8 +523,8 @@ static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo)
primary = primary_cpumask; primary = primary_cpumask;
if (!primary) if (!primary)
return -ENOENT; return -ENOENT;
turbo = turbo_cpumask; preferred = preferred_cpumask;
if (!turbo) if (!preferred)
return -ENOENT; return -ENOENT;
/* /*
@ -594,16 +594,16 @@ static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo)
/* /*
* Determine the task's scheduling domain. * Determine the task's scheduling domain.
* *
* Try to dispatch on the turbo boosted CPUs first. If we can't find * Try to dispatch on the preferred CPUs first. If we can't find any
* any idle CPU, re-try again with the primary scheduling domain. * idle CPU, re-try again with the primary scheduling domain.
*/ */
if (do_turbo && if (do_preferred &&
!bpf_cpumask_empty(cast_mask(turbo)) && !bpf_cpumask_empty(cast_mask(preferred)) &&
!bpf_cpumask_equal(cast_mask(turbo), cast_mask(primary))) { !bpf_cpumask_equal(cast_mask(preferred), cast_mask(primary))) {
bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(turbo)); bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(preferred));
} else { } else {
bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(primary)); bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(primary));
do_turbo = false; do_preferred = false;
} }
/* /*
@ -666,10 +666,10 @@ static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo)
} }
/* /*
* When considering the turbo domain (first idle CPU selection * When considering the preferred domain (first idle CPU
* pass) try to stay on the same LLC. * selection pass) try to stay on the same LLC.
*/ */
if (do_turbo) { if (do_preferred) {
cpu = -ENOENT; cpu = -ENOENT;
goto out_put_cpumask; goto out_put_cpumask;
} }
@ -716,10 +716,10 @@ static s32 pick_idle_cpu(struct task_struct *p, s32 prev_cpu, bool do_turbo)
} }
/* /*
* When considering the turbo domain (first idle CPU selection pass) * When considering the preferred domain (first idle CPU selection
* try to stay on the same LLC. * pass) try to stay on the same LLC.
*/ */
if (do_turbo) { if (do_preferred) {
cpu = -ENOENT; cpu = -ENOENT;
goto out_put_cpumask; goto out_put_cpumask;
} }
@ -1351,20 +1351,20 @@ int enable_sibling_cpu(struct domain_arg *input)
} }
SEC("syscall") SEC("syscall")
int enable_turbo_cpu(struct cpu_arg *input) int enable_preferred_cpu(struct cpu_arg *input)
{ {
struct bpf_cpumask *mask; struct bpf_cpumask *mask;
int err = 0; int err = 0;
/* Make sure the primary CPU mask is initialized */ /* Make sure the primary CPU mask is initialized */
err = init_cpumask(&turbo_cpumask); err = init_cpumask(&preferred_cpumask);
if (err) if (err)
return err; return err;
/* /*
* Enable the target CPU in the turbo boost scheduling domain. * Enable the target CPU in the preferred scheduling domain.
*/ */
bpf_rcu_read_lock(); bpf_rcu_read_lock();
mask = turbo_cpumask; mask = preferred_cpumask;
if (mask) { if (mask) {
s32 cpu = input->cpu_id; s32 cpu = input->cpu_id;
@ -1465,8 +1465,8 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(bpfland_init)
if (err) if (err)
return err; return err;
/* Initialize the primary scheduling domain */ /* Initialize the preferred scheduling domain */
err = init_cpumask(&turbo_cpumask); err = init_cpumask(&preferred_cpumask);
if (err) if (err)
return err; return err;

62
scheds/rust/scx_bpfland/src/main.rs
View File

@ -164,17 +164,26 @@ struct Opts {
#[clap(short = 'k', long, action = clap::ArgAction::SetTrue)] #[clap(short = 'k', long, action = clap::ArgAction::SetTrue)]
local_kthreads: bool, local_kthreads: bool,
/// Specifies a group of preferred CPUs, represented as a bitmask in hex (e.g., 0xff), that the
/// scheduler will try to prioritize to dispatch tasks.
///
/// Special values:
/// - "auto" = automaticlly detect the fastest CPUs based on the current scheduler and system
/// energy profiles.
#[clap(short = 'M', long, default_value = "auto")]
preferred_domain: String,
/// Specifies the initial set of CPUs, represented as a bitmask in hex (e.g., 0xff), that the /// Specifies the initial set of CPUs, represented as a bitmask in hex (e.g., 0xff), that the
/// scheduler will use to dispatch tasks, until the system becomes saturated, at which point /// scheduler will use to dispatch tasks, until the system becomes saturated, at which point
/// tasks may overflow to other available CPUs. /// tasks may overflow to other available CPUs.
/// ///
/// Special values: /// Special values:
/// - "auto" = automatically detect the CPUs based on the current energy profile
/// - "performance" = automatically detect and prioritize the fastest CPUs /// - "performance" = automatically detect and prioritize the fastest CPUs
/// - "powersave" = automatically detect and prioritize the slowest CPUs /// - "powersave" = automatically detect and prioritize the slowest CPUs
/// - "auto" = automatically detect the CPUs based on the current energy profile
/// - "all" = all CPUs assigned to the primary domain /// - "all" = all CPUs assigned to the primary domain
/// - "none" = no prioritization, tasks are dispatched on the first CPU available /// - "none" = no prioritization, tasks are dispatched on the first CPU available
#[clap(short = 'm', long, default_value = "all")] #[clap(short = 'm', long, default_value = "auto")]
primary_domain: String, primary_domain: String,
/// Disable L2 cache awareness. /// Disable L2 cache awareness.
@ -284,11 +293,11 @@ impl<'a> Scheduler<'a> {
// Initialize CPU topology. // Initialize CPU topology.
let topo = Topology::new().unwrap(); let topo = Topology::new().unwrap();
// Initialize the primary scheduling domain (based on the --primary-domain option). // Initialize the primary scheduling domain and the preferred domain.
let energy_profile = Self::read_energy_profile(); let energy_profile = Self::read_energy_profile();
if let Err(err) = Self::init_turbo_domain(&mut skel, &opts.primary_domain) if let Err(err) = Self::init_preferred_domain(&mut skel, &opts.preferred_domain)
{ {
warn!("failed to initialize turbo domain: error {}", err); warn!("failed to initialize preferred domain: error {}", err);
} }
if let Err(err) = Self::init_energy_domain(&mut skel, &opts.primary_domain, &energy_profile) if let Err(err) = Self::init_energy_domain(&mut skel, &opts.primary_domain, &energy_profile)
{ {
@ -371,8 +380,8 @@ impl<'a> Scheduler<'a> {
res.unwrap_or_else(|_: String| "none".to_string()) res.unwrap_or_else(|_: String| "none".to_string())
} }
fn enable_turbo_cpu(skel: &mut BpfSkel<'_>, cpu: i32) -> Result<(), u32> { fn enable_preferred_cpu(skel: &mut BpfSkel<'_>, cpu: i32) -> Result<(), u32> {
let prog = &mut skel.progs.enable_turbo_cpu; let prog = &mut skel.progs.enable_preferred_cpu;
let mut args = cpu_arg { let mut args = cpu_arg {
cpu_id: cpu as c_int, cpu_id: cpu as c_int,
}; };
@ -401,29 +410,29 @@ impl<'a> Scheduler<'a> {
Cpumask::from_str(&cpus_to_cpumask(&cpus)) Cpumask::from_str(&cpus_to_cpumask(&cpus))
} }
fn init_turbo_domain( fn init_preferred_domain(
skel: &mut BpfSkel<'_>, skel: &mut BpfSkel<'_>,
primary_domain: &String, preferred_domain: &String,
) -> Result<()> { ) -> Result<()> {
let domain = match primary_domain.as_str() { let domain = match preferred_domain.as_str() {
"auto" => { "auto" => {
Self::epp_to_cpumask(Powermode::Turbo)? Self::epp_to_cpumask(Powermode::Turbo)?
} }
&_ => { &_ => {
Cpumask::new()? Cpumask::from_str(&preferred_domain)?
} }
}; };
info!("Turbo CPU domain = 0x{:x}", domain); info!("preferred CPU domain = 0x{:x}", domain);
// Clear the turbo domain by passing a negative CPU id. // Clear the preferred domain by passing a negative CPU id.
if let Err(err) = Self::enable_turbo_cpu(skel, -1) { if let Err(err) = Self::enable_preferred_cpu(skel, -1) {
warn!("failed to reset primary domain: error {}", err); warn!("failed to reset preferred domain: error {}", err);
} }
for cpu in 0..*NR_CPU_IDS { for cpu in 0..*NR_CPU_IDS {
if domain.test_cpu(cpu) { if domain.test_cpu(cpu) {
if let Err(err) = Self::enable_turbo_cpu(skel, cpu as i32) { if let Err(err) = Self::enable_preferred_cpu(skel, cpu as i32) {
warn!("failed to add CPU {} to turbo domain: error {}", cpu, err); warn!("failed to add CPU {} to preferred domain: error {}", cpu, err);
} }
} }
} }
@ -490,9 +499,9 @@ impl<'a> Scheduler<'a> {
) -> Result<()> { ) -> Result<()> {
let perf_lvl: i64 = match primary_domain.as_str() { let perf_lvl: i64 = match primary_domain.as_str() {
"auto" => match energy_profile.as_str() { "auto" => match energy_profile.as_str() {
"performance" => 1024,
"power" | "powersave" => 0, "power" | "powersave" => 0,
"balance_power" => -1, &_ => -1,
&_ => 1024,
}, },
"performance" => 1024, "performance" => 1024,
"powersave" => 0, "powersave" => 0,
@ -500,10 +509,11 @@ impl<'a> Scheduler<'a> {
}; };
info!( info!(
"cpufreq performance level: {}", "cpufreq performance level: {}",
if perf_lvl < 0 { match perf_lvl {
"auto".into() 1024 => "max".into(),
} else { 0 => "min".into(),
perf_lvl.to_string() n if n < 0 => "auto".into(),
_ => perf_lvl.to_string(),
} }
); );
skel.maps.bss_data.cpufreq_perf_lvl = perf_lvl; skel.maps.bss_data.cpufreq_perf_lvl = perf_lvl;
@ -518,11 +528,11 @@ impl<'a> Scheduler<'a> {
self.energy_profile = energy_profile.clone(); self.energy_profile = energy_profile.clone();
if self.opts.primary_domain == "auto" { if self.opts.primary_domain == "auto" {
if let Err(err) = Self::init_turbo_domain( if let Err(err) = Self::init_preferred_domain(
&mut self.skel, &mut self.skel,
&self.opts.primary_domain, &self.opts.preferred_domain,
) { ) {
warn!("failed to refresh turbo domain: error {}", err); warn!("failed to refresh preferred domain: error {}", err);
} }
if let Err(err) = Self::init_energy_domain( if let Err(err) = Self::init_energy_domain(
&mut self.skel, &mut self.skel,