JakeHillion/scx
1
0
Fork 0
mirror of https://github.com/JakeHillion/scx.git synced 2024-11-29 12:40:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #861 from multics69/lavd-opt-dispatch

Browse Source 
scx_lavd: tuning and optimizing latency criticality calculation
This commit is contained in:
Changwoo Min 2024-10-31 08:10:21 +09:00 committed by GitHub
commit 83b5f4eb23
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 45 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
scheds/rust/scx_lavd/src/bpf/main.bpf.c
View File

@ -263,7 +263,7 @@ static u64 calc_freq_factor(u64 freq, u64 weight_ft)
}
static u64 calc_weight_factor(struct task_struct *p, struct task_ctx *taskc,
struct cpu_ctx *cpuc_cur, u64 enq_flags)
u64 enq_flags)
{
u64 weight_boost = 1;
u64 weight_ft;
@ -283,10 +283,10 @@ static u64 calc_weight_factor(struct task_struct *p, struct task_ctx *taskc,
weight_boost += LAVD_LC_WEIGHT_BOOST;
/*
* Reset task's lock and futex boost count
* for a lock holder to be boosted only once.
* A pinned-task tends to be latency-critical.
*/
reset_lock_futex_boost(taskc, cpuc_cur);
if (p->nr_cpus_allowed == 1)
weight_boost += LAVD_LC_WEIGHT_BOOST;
/*
* Prioritize a lock holder for faster system-wide forward progress.
@ -301,7 +301,7 @@ static u64 calc_weight_factor(struct task_struct *p, struct task_ctx *taskc,
}
static u64 calc_lat_cri(struct task_struct *p, struct task_ctx *taskc,
struct cpu_ctx *cpuc_cur, u64 enq_flags)
u64 enq_flags)
{
u64 weight_ft, wait_freq_ft, wake_freq_ft, runtime_ft;
u64 lat_cri;
@ -310,7 +310,7 @@ static u64 calc_lat_cri(struct task_struct *p, struct task_ctx *taskc,
* Adjust task's weight based on the scheduling context, such as
* if it is a kernel task, lock holder, etc.
*/
weight_ft = calc_weight_factor(p, taskc, cpuc_cur, enq_flags);
weight_ft = calc_weight_factor(p, taskc, enq_flags);
/*
* A task is more latency-critical as its wait or wake frequencies
@ -355,7 +355,6 @@ static u64 calc_lat_cri(struct task_struct *p, struct task_ctx *taskc,
static void calc_virtual_deadline_delta(struct task_struct *p,
struct task_ctx *taskc,
struct cpu_ctx *cpuc_cur,
u64 enq_flags)
{
u64 deadline, lat_cri;
@ -364,7 +363,7 @@ static void calc_virtual_deadline_delta(struct task_struct *p,
/*
* Calculate the deadline based on latency criticality and greedy ratio.
*/
lat_cri = calc_lat_cri(p, taskc, cpuc_cur, enq_flags);
lat_cri = calc_lat_cri(p, taskc, enq_flags);
greedy_ratio = calc_greedy_ratio(taskc);
greedy_ft = calc_greedy_factor(greedy_ratio);
@ -448,8 +447,7 @@ static u64 get_suspended_duration_and_reset(struct cpu_ctx *cpuc)
}
static void update_stat_for_runnable(struct task_struct *p,
struct task_ctx *taskc,
struct cpu_ctx *cpuc)
struct task_ctx *taskc)
{
/*
* Reflect task's load immediately.
@ -515,10 +513,12 @@ static void update_stat_for_running(struct task_struct *p,
cpuc->sum_lat_cri += taskc->lat_cri;
cpuc->nr_sched++;
/*
* Update lock holder information on the cpu.
* Reset task's lock and futex boost count
* for a lock holder to be boosted only once.
*/
cpuc->lock_holder = is_lock_holder(taskc);
reset_lock_futex_boost(taskc, cpuc);
/*
* It is clear there is no need to consider the suspended duration
@ -938,12 +938,26 @@ static void update_task_log_clk(struct task_ctx *taskc)
WRITE_ONCE(taskc->vdeadline_log_clk, vlc);
}
static void direct_dispatch(struct task_struct *p, struct task_ctx *taskc)
static void direct_dispatch(struct task_struct *p, struct task_ctx *taskc,
u64 enq_flags)
{
/*
* Calculate latency criticality for preemptability test.
*/
calc_lat_cri(p, taskc, 0);
/*
* Reset the vdeadline_delta_ns to update the task's logical clock.
*/
taskc->vdeadline_delta_ns = 0;
update_task_log_clk(taskc);
/*
* Calculate the duration to run.
*/
p->scx.slice = calc_time_slice(p, taskc);
scx_bpf_dispatch(p, SCX_DSQ_LOCAL, p->scx.slice, 0);
scx_bpf_dispatch(p, SCX_DSQ_LOCAL, p->scx.slice, enq_flags);
}
s32 BPF_STRUCT_OPS(lavd_select_cpu, struct task_struct *p, s32 prev_cpu,
@ -956,26 +970,32 @@ s32 BPF_STRUCT_OPS(lavd_select_cpu, struct task_struct *p, s32 prev_cpu,
taskc = get_task_ctx(p);
if (!taskc)
return prev_cpu;
taskc->wakeup_ft += !!(wake_flags & SCX_WAKE_SYNC);
cpu_id = pick_idle_cpu(p, taskc, prev_cpu, wake_flags, &found_idle);
if (found_idle) {
direct_dispatch(p, taskc);
/*
* If there is an idle cpu,
* disptach the task to the idle cpu right now.
*/
direct_dispatch(p, taskc, 0);
return cpu_id;
}
taskc->wakeup_ft += !!(wake_flags & SCX_WAKE_SYNC);
/*
* Even if there is no idle cpu, still repect the chosen cpu.
*/
return cpu_id >= 0 ? cpu_id : prev_cpu;
}
static void calc_when_to_run(struct task_struct *p, struct task_ctx *taskc,
struct cpu_ctx *cpuc_cur, u64 enq_flags)
u64 enq_flags)
{
/*
* Before enqueueing a task to a run queue, we should decide when a
* task should be scheduled.
*/
calc_virtual_deadline_delta(p, taskc, cpuc_cur, enq_flags);
calc_virtual_deadline_delta(p, taskc, enq_flags);
update_task_log_clk(taskc);
}
@ -1042,11 +1062,8 @@ void BPF_STRUCT_OPS(lavd_enqueue, struct task_struct *p, u64 enq_flags)
/*
* Calculate when a tack can be scheduled.
*
* Note that the task's time slice will be calculated and reassigned
* right before running at ops.running().
*/
calc_when_to_run(p, taskc, cpuc_cur, enq_flags);
calc_when_to_run(p, taskc, enq_flags);
dsq_id = find_proper_dsq(taskc, cpuc_task);
/*
@ -1058,7 +1075,7 @@ void BPF_STRUCT_OPS(lavd_enqueue, struct task_struct *p, u64 enq_flags)
* Enqueue the task to one of task's DSQs based on its virtual deadline.
*/
scx_bpf_dispatch_vtime(p, dsq_id, p->scx.slice,
taskc->vdeadline_log_clk, enq_flags);
taskc->vdeadline_log_clk, enq_flags);
/*
* If there is an idle cpu for the task, try to kick it up now
@ -1223,7 +1240,7 @@ void BPF_STRUCT_OPS(lavd_dispatch, s32 cpu, struct task_struct *prev)
cpuc = get_cpu_ctx_id(cpu);
if (!cpuc) {
scx_bpf_error("Failed to look up cpu context or task context");
scx_bpf_error("Failed to look up cpu context context");
return;
}
@ -1410,7 +1427,6 @@ update_cpuperf:
void BPF_STRUCT_OPS(lavd_runnable, struct task_struct *p, u64 enq_flags)
{
struct cpu_ctx *cpuc;
struct task_struct *waker;
struct task_ctx *p_taskc, *waker_taskc;
u64 now, interval;
@ -1420,12 +1436,11 @@ void BPF_STRUCT_OPS(lavd_runnable, struct task_struct *p, u64 enq_flags)
* rq. Statistics will be adjusted when more accurate statistics become
* available (ops.running).
*/
cpuc = get_cpu_ctx();
p_taskc = get_task_ctx(p);
if (!cpuc || !p_taskc)
if (!p_taskc)
return;
update_stat_for_runnable(p, p_taskc, cpuc);
update_stat_for_runnable(p, p_taskc);
/*
* When a task @p is wakened up, the wake frequency of its waker task

2
scheds/rust/scx_lavd/src/bpf/preempt.bpf.c
View File

@ -247,7 +247,7 @@ static bool try_find_and_kick_victim_cpu(struct task_struct *p,
bool ret = false;
/*
* Prepare a cpumak so we find a victim @p's compute domain.
* Prepare a cpumak so we find a victim in @p's compute domain.
*/
cpumask = cpuc_cur->tmp_t_mask;
cpdomc = MEMBER_VPTR(cpdom_ctxs, [dsq_id]);