JakeHillion/scx
1
0
Fork 0
mirror of https://github.com/JakeHillion/scx.git synced 2024-11-25 11:00:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

scx_layered: Use cached cpu_ctx->hi_fallback_dsq_id and cpu_ctx->cached_idx

Browse Source 
- Remember hi_fallback_dsq_id for each CPU in cpu_ctx and use the remembered
  values.

- Make antistall_scan() walk each hi fallback DSQ once instead of multiple
  times through CPU iteration.

- Remove unused functions.
This commit is contained in:
Tejun Heo 2024-11-18 10:09:24 -10:00
parent 827af0b7ef
commit 68e1741351
2 changed files with 23 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scheds/rust/scx_layered/src/bpf/intf.h
View File

@ -112,6 +112,7 @@ struct cpu_ctx {
u64 gstats[NR_GSTATS];
u64 lstats[MAX_LAYERS][NR_LSTATS];
u64 ran_current_for;
u64 hi_fallback_dsq_id;
u32 layer_idx;
u32 cache_idx;
u32 node_idx;

87
scheds/rust/scx_layered/src/bpf/main.bpf.c
View File

@ -118,12 +118,9 @@ static __noinline u64 layer_dsq_id(u32 layer_id, u32 llc_id)
return (layer_id * nr_llcs) + llc_id;
}
// XXX - cpu_to_llc_id() must not be inlined to not blow past ins limit when
// topo is enabled but older kernels get confused by RCU state when subprogs are
// called from sleepable functions. Use __always_inline variant from
// layered_init() and __noinline from everywhere else. Remove this once we can
// ignore the older kernels.
static __always_inline u32 __cpu_to_llc_id(s32 cpu_id)
// XXX - older kernels get confused by RCU state when subprogs are called from
// sleepable functions. Use __always_inline.
static __always_inline u32 cpu_to_llc_id(s32 cpu_id)
{
const volatile u32 *llc_ptr;
@ -135,11 +132,6 @@ static __always_inline u32 __cpu_to_llc_id(s32 cpu_id)
return *llc_ptr;
}
static __noinline u32 cpu_to_llc_id(u32 cpu_id)
{
return __cpu_to_llc_id(cpu_id);
}
u32 llc_node_id(u32 llc_id)
{
const volatile u32 *llc_ptr;
@ -162,33 +154,6 @@ static inline bool is_fallback_dsq(u64 dsq_id)
return dsq_id > HI_FALLBACK_DSQ_BASE && dsq_id <= LO_FALLBACK_DSQ;
}
static u64 llc_hi_fallback_dsq_iter_offset(int llc_offset, int idx)
{
int offset = llc_offset + idx;
if (offset >= nr_llcs)
return llc_hi_fallback_dsq_id(offset - nr_llcs);
return llc_hi_fallback_dsq_id(idx + llc_offset);
}
static int llc_iter_cpu_offset(int idx, s32 cpu)
{
int offset;
if (cpu <= 0)
return idx;
offset = (cpu % nr_llcs) + idx;
return offset >= nr_llcs ? offset - nr_llcs : offset;
}
static u64 cpu_hi_fallback_dsq_id(s32 cpu)
{
return llc_hi_fallback_dsq_id(cpu_to_llc_id(cpu));
}
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__type(key, u32);
@ -1095,16 +1060,15 @@ preempt_fail:
void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
{
struct cpu_ctx *cctx;
struct cpu_ctx *cctx, *task_cctx;
struct task_ctx *tctx;
struct layer *layer;
s32 task_cpu = scx_bpf_task_cpu(p);
u64 vtime = p->scx.dsq_vtime;
bool try_preempt_first;
u32 idx;
if (!(cctx = lookup_cpu_ctx(-1)) || !(tctx = lookup_task_ctx(p)) ||
!(layer = lookup_layer(tctx->layer)))
if (!(cctx = lookup_cpu_ctx(-1)) || !(task_cctx = lookup_cpu_ctx(task_cpu)) ||
!(tctx = lookup_task_ctx(p)) || !(layer = lookup_layer(tctx->layer)))
return;
try_preempt_first = cctx->try_preempt_first;
@ -1147,9 +1111,8 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
!bpf_cpumask_test_cpu(task_cpu, layer_cpumask))
lstat_inc(LSTAT_AFFN_VIOL, layer, cctx);
idx = cpu_hi_fallback_dsq_id(task_cpu);
tctx->last_dsq = idx;
scx_bpf_dispatch(p, idx, slice_ns, enq_flags);
tctx->last_dsq = task_cctx->hi_fallback_dsq_id;
scx_bpf_dispatch(p, tctx->last_dsq, slice_ns, enq_flags);
goto preempt;
}
@ -1173,9 +1136,8 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
* to the LLC local HI_FALLBACK_DSQ to avoid this starvation
* issue.
*/
idx = cpu_hi_fallback_dsq_id(task_cpu);
scx_bpf_dispatch(p, idx, slice_ns, enq_flags);
tctx->last_dsq = idx;
tctx->last_dsq = task_cctx->hi_fallback_dsq_id;
scx_bpf_dispatch(p, tctx->last_dsq, slice_ns, enq_flags);
goto preempt;
}
@ -1183,9 +1145,8 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
tctx->last_dsq = tctx->layer;
scx_bpf_dispatch_vtime(p, tctx->layer, slice_ns, vtime, enq_flags);
} else {
idx = layer_dsq_id(layer->idx, cpu_to_llc_id(task_cpu));
tctx->last_dsq = idx;
scx_bpf_dispatch_vtime(p, idx, slice_ns, vtime, enq_flags);
tctx->last_dsq = layer_dsq_id(layer->idx, task_cctx->cache_idx);
scx_bpf_dispatch_vtime(p, tctx->last_dsq, slice_ns, vtime, enq_flags);
}
preempt:
@ -1239,8 +1200,7 @@ static bool keep_running(struct cpu_ctx *cctx, struct task_struct *p)
return true;
}
} else {
u32 dsq_id = layer_dsq_id(layer->idx,
cpu_to_llc_id(bpf_get_smp_processor_id()));
u32 dsq_id = layer_dsq_id(layer->idx, cctx->cache_idx);
if (!scx_bpf_dsq_nr_queued(dsq_id)) {
p->scx.slice = slice_ns;
lstat_inc(LSTAT_KEEP, layer, cctx);
@ -1448,8 +1408,7 @@ void layered_dispatch_no_topo(s32 cpu, struct task_struct *prev)
return;
}
dsq_id = cpu_hi_fallback_dsq_id(cpu);
if (scx_bpf_consume(dsq_id))
if (scx_bpf_consume(cctx->hi_fallback_dsq_id))
return;
/* consume !open layers second */
@ -1708,8 +1667,6 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)
return;
}
u32 my_llc_id = cpu_to_llc_id(cpu);
/*
* If one of the fallback DSQs has the most budget then consume from it
* to prevent starvation.
@ -1724,19 +1681,18 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)
}
/* consume preempting layers first */
if (consume_preempting(costc, my_llc_id) == 0)
if (consume_preempting(costc, cctx->cache_idx) == 0)
return;
dsq_id = cpu_hi_fallback_dsq_id(cpu);
if (scx_bpf_consume(dsq_id))
if (scx_bpf_consume(cctx->hi_fallback_dsq_id))
return;
/* consume !open layers second */
if (consume_non_open(costc, cpu, my_llc_id) == 0)
if (consume_non_open(costc, cpu, cctx->cache_idx) == 0)
return;
/* consume !preempting open layers */
if (consume_open_no_preempt(costc, my_llc_id) == 0)
if (consume_open_no_preempt(costc, cctx->cache_idx) == 0)
return;
scx_bpf_consume(LO_FALLBACK_DSQ);
@ -2019,13 +1975,14 @@ static s32 create_cache(u32 cache_id)
return -ENOENT;
}
llc_id = __cpu_to_llc_id(cpu);
llc_id = cpu_to_llc_id(cpu);
if (llc_id != cache_id)
continue;
bpf_cpumask_set_cpu(cpu, cpumask);
cachec->nr_cpus++;
cctx->cache_idx = cache_id;
cctx->hi_fallback_dsq_id = llc_hi_fallback_dsq_id(cache_id);
}
dbg("CFG creating cache %d with %d cpus", cache_id, cachec->nr_cpus);
@ -2614,8 +2571,8 @@ static bool antistall_scan(void)
bpf_for(llc, 0, nr_llcs)
antistall_set(layer_dsq_id(layer_id, llc), jiffies_now);
bpf_for(cpu, 0, nr_possible_cpus)
antistall_set(cpu_hi_fallback_dsq_id(cpu), jiffies_now);
bpf_for(llc, 0, nr_llcs)
antistall_set(llc_hi_fallback_dsq_id(llc), jiffies_now);
antistall_set(LO_FALLBACK_DSQ, jiffies_now);