JakeHillion/scx
1
0
Fork 0
mirror of https://github.com/JakeHillion/scx.git synced 2024-11-26 11:30:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

scx_lavd: drop 2-level-scheduling

Browse Source 
With optimizations of calculatring ineligibility duration, now the
scheduler works well under heavy load without 2-level scheduling, so we
drop it for simplicitiy.

Signed-off-by: Changwoo Min <changwoo@igalia.com>
This commit is contained in:
Changwoo Min 2024-08-02 21:46:07 +09:00
parent c38e749c36
commit f3fd6e9cb3
3 changed files with 11 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scheds/rust/scx_lavd/src/bpf/intf.h
View File

@ -91,9 +91,7 @@ enum consts {
LAVD_CC_CPU_PIN_INTERVAL_DIV = (LAVD_CC_CPU_PIN_INTERVAL /
LAVD_SYS_STAT_INTERVAL_NS),
LAVD_LATENCY_CRITICAL_DSQ = 0, /* a global DSQ for latency-criticcal tasks */
LAVD_REGULAR_DSQ = 1, /* a global DSQ for non-latency-criticcal tasks */
LAVD_DSQ_STARVE_TIMEOUT = (5ULL * NSEC_PER_USEC),
LAVD_GLOBAL_DSQ = 0, /* a global DSQ */
LAVD_STATUS_STR_LEN = 5, /* {LR: Latency-critical, Regular}
{HI: performance-Hungry, performance-Insensitive}

123
scheds/rust/scx_lavd/src/bpf/main.bpf.c
View File

@ -224,17 +224,10 @@ static u64 cur_logical_clk;
*/
static u64 cur_svc_time;
/*
* Last task consumption time
*/
static u64 lat_cri_rq_clk; /* last task consumption timem for latency-critical task */
static u64 regular_rq_clk; /* last task consumption timem for latency-critical task */
/*
* Options
*/
const volatile bool no_core_compaction;
const volatile bool no_2_level_scheduling;
const volatile bool no_freq_scaling;
const volatile u8 verbose;
@ -503,8 +496,6 @@ static bool have_scheduled(struct task_ctx *taskc)
static void try_proc_introspec_cmd(struct task_struct *p,
struct task_ctx *taskc, u32 cpu_id)
{
bool ret;
if (LAVD_CPU_ID_HERE == cpu_id)
cpu_id = bpf_get_smp_processor_id();
@ -1115,8 +1106,7 @@ static u64 clamp_time_slice_ns(u64 slice)
static s32 nr_queued_tasks(void)
{
return scx_bpf_dsq_nr_queued(LAVD_LATENCY_CRITICAL_DSQ) +
scx_bpf_dsq_nr_queued(LAVD_REGULAR_DSQ);
return scx_bpf_dsq_nr_queued(LAVD_GLOBAL_DSQ);
}
static u64 calc_time_slice(struct task_struct *p, struct task_ctx *taskc,
@ -1663,7 +1653,7 @@ static bool try_yield_current_cpu(struct task_struct *p_run,
prm_run.lat_cri = taskc_run->lat_cri;
bpf_rcu_read_lock();
bpf_for_each(scx_dsq, p_wait, LAVD_LATENCY_CRITICAL_DSQ, 0) {
bpf_for_each(scx_dsq, p_wait, LAVD_GLOBAL_DSQ, 0) {
taskc_wait = get_task_ctx(p_wait);
if (!taskc_wait)
break;
@ -1688,7 +1678,7 @@ static bool try_yield_current_cpu(struct task_struct *p_run,
}
/*
* Test only the first entry on the LAVD_LATENCY_CRITICAL_DSQ.
* Test only the first entry on the LAVD_GLOBAL_DSQ.
*/
break;
}
@ -1709,7 +1699,6 @@ static void put_global_rq(struct task_struct *p, struct task_ctx *taskc,
{
struct task_ctx *taskc_run;
struct task_struct *p_run;
u64 dsq_id = LAVD_REGULAR_DSQ;
/*
* Calculate when a tack can be scheduled.
@ -1740,14 +1729,8 @@ static void put_global_rq(struct task_struct *p, struct task_ctx *taskc,
/*
* Enqueue the task to one of the DSQs based on its virtual deadline.
*
* Note that, with no_2_level_scheduling, all tasks are considered
* latency-critical and they're all enqueed to the
* LAVD_LATENCY_CRITICAL_DSQ.
*/
if (no_2_level_scheduling || is_lat_cri_task(taskc))
dsq_id = LAVD_LATENCY_CRITICAL_DSQ;
scx_bpf_dispatch_vtime(p, dsq_id, LAVD_SLICE_UNDECIDED,
scx_bpf_dispatch_vtime(p, LAVD_GLOBAL_DSQ, LAVD_SLICE_UNDECIDED,
taskc->vdeadline_log_clk, enq_flags);
return;
}
@ -1923,67 +1906,16 @@ static bool use_full_cpus(void)
((stat_cur->nr_active + LAVD_CC_NR_OVRFLW) >= nr_cpus_onln);
}
static __always_inline
bool consume_lat_cri_task(u64 now)
{
if (scx_bpf_consume(LAVD_LATENCY_CRITICAL_DSQ)) {
WRITE_ONCE(lat_cri_rq_clk, now);
return true;
}
return false;
}
static __always_inline
bool consume_regular_task(u64 now)
{
if (scx_bpf_consume(LAVD_REGULAR_DSQ)) {
WRITE_ONCE(regular_rq_clk, now);
return true;
}
return false;
}
static __always_inline
bool consume_starving_task(u64 now)
{
u64 clk;
clk = READ_ONCE(lat_cri_rq_clk) + LAVD_DSQ_STARVE_TIMEOUT;
if (clk < now && consume_lat_cri_task(now))
return true;
clk = READ_ONCE(regular_rq_clk) + LAVD_DSQ_STARVE_TIMEOUT;
if (clk < now && consume_regular_task(now))
return true;
return false;
}
static __always_inline
bool consume_task(u64 now)
{
if (!no_2_level_scheduling && consume_starving_task(now))
return true;
if (consume_lat_cri_task(now))
return true;
if (!no_2_level_scheduling && consume_regular_task(now))
return true;
return false;
}
void BPF_STRUCT_OPS(lavd_dispatch, s32 cpu, struct task_struct *prev)
{
u64 now = bpf_ktime_get_ns();
struct bpf_cpumask *active, *ovrflw;
struct task_struct *p;
bool ret = false;
/*
* If all CPUs are using, directly consume without checking CPU masks.
*/
if (use_full_cpus()) {
consume_task(now);
scx_bpf_consume(LAVD_GLOBAL_DSQ);
return;
}
@ -2004,7 +1936,7 @@ void BPF_STRUCT_OPS(lavd_dispatch, s32 cpu, struct task_struct *prev)
*/
if (bpf_cpumask_test_cpu(cpu, cast_mask(active)) ||
bpf_cpumask_test_cpu(cpu, cast_mask(ovrflw))) {
consume_task(now);
scx_bpf_consume(LAVD_GLOBAL_DSQ);
goto unlock_out;
}
@ -2012,14 +1944,14 @@ void BPF_STRUCT_OPS(lavd_dispatch, s32 cpu, struct task_struct *prev)
* If this CPU is not either in active or overflow CPUs, it tries to
* find and run a task pinned to run on this CPU.
*/
bpf_for_each(scx_dsq, p, LAVD_LATENCY_CRITICAL_DSQ, 0) {
bpf_for_each(scx_dsq, p, LAVD_GLOBAL_DSQ, 0) {
/*
* Prioritize kernel tasks because most kernel tasks are pinned
* to a particular CPU and latency-critical (e.g., ksoftirqd,
* kworker, etc).
*/
if (is_kernel_task(p)) {
ret = consume_task(now);
scx_bpf_consume(LAVD_GLOBAL_DSQ);
bpf_cpumask_set_cpu(cpu, ovrflw);
break;
}
@ -2051,7 +1983,7 @@ void BPF_STRUCT_OPS(lavd_dispatch, s32 cpu, struct task_struct *prev)
* cores. We will optimize this path after introducing per-core
* DSQ.
*/
ret = consume_task(now);
scx_bpf_consume(LAVD_GLOBAL_DSQ);
/*
* This is the first time a particular pinned user-space task
@ -2068,35 +2000,6 @@ release_break:
break;
}
/*
* With no_2_level_scheduling, all tasks are considered
* latency-critical, so we don't need to check the regular quque.
*/
if (no_2_level_scheduling || ret)
goto unlock_out;
bpf_for_each(scx_dsq, p, LAVD_REGULAR_DSQ, 0) {
if (is_kernel_task(p)) {
consume_task(now);
bpf_cpumask_set_cpu(cpu, ovrflw);
break;
}
p = bpf_task_from_pid(p->pid);
if (!p)
goto unlock_out;
if (bpf_cpumask_intersects(cast_mask(active), p->cpus_ptr) ||
bpf_cpumask_intersects(cast_mask(ovrflw), p->cpus_ptr))
goto release_break2;
consume_task(now);
bpf_cpumask_set_cpu(cpu, ovrflw);
release_break2:
bpf_task_release(p);
break;
}
unlock_out:
bpf_rcu_read_unlock();
return;
@ -2590,18 +2493,12 @@ static s32 init_dsqs(void)
{
int err;
err = scx_bpf_create_dsq(LAVD_LATENCY_CRITICAL_DSQ, -1);
err = scx_bpf_create_dsq(LAVD_GLOBAL_DSQ, -1);
if (err) {
scx_bpf_error("Failed to create a latency critical DSQ");
return err;
}
err = scx_bpf_create_dsq(LAVD_REGULAR_DSQ, -1);
if (err) {
scx_bpf_error("Failed to create a regular DSQ");
return err;
}
return 0;
}

4
scheds/rust/scx_lavd/src/main.rs
View File

@ -60,10 +60,6 @@ struct Opts {
#[clap(long = "prefer-smt-core", action = clap::ArgAction::SetTrue)]
prefer_smt_core: bool,
/// Disable 2-level scheduling, which segregates latency-critical tasks from regular tasks
#[clap(long = "no-2-level-scheduling", action = clap::ArgAction::SetTrue)]
no_2_level_scheduling: bool,
/// Disable frequency scaling by scx_lavd
#[clap(long = "no-freq-scaling", action = clap::ArgAction::SetTrue)]
no_freq_scaling: bool,