JakeHillion/scx-upstream
1
0
Fork 0
mirror of https://github.com/sched-ext/scx.git synced 2024-11-21 18:41:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 4fc0509178 into 72f21dba06

Browse Source
This commit is contained in:
Daniel Hodges 2024-11-13 20:44:01 +00:00 committed by GitHub
commit 21260e5f36
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 126 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
scheds/rust/scx_layered/src/bpf/main.bpf.c
View File

@ -37,6 +37,7 @@ const volatile bool smt_enabled = true;
const volatile bool has_little_cores = true;
const volatile bool disable_topology = false;
const volatile bool xnuma_preemption = false;
const volatile bool local_llc_iteration = true;
const volatile s32 __sibling_cpu[MAX_CPUS];
const volatile bool monitor_disable = false;
const volatile unsigned char all_cpus[MAX_CPUS_U8];
@ -1467,28 +1468,47 @@ __weak int consume_preempting(struct cost *costc, u32 my_llc_id)
if (!costc)
return -EINVAL;
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (!layer->preempt || has_budget(costc, layer) == 0)
continue;
if (local_llc_iteration) {
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
} else {
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (!layer->preempt || has_budget(costc, layer) == 0)
continue;
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
}
return -ENOENT;
}
__weak int consume_non_open(struct cost *costc, s32 cpu, u32 my_llc_id)
static __noinline int consume_non_open(struct cost *costc, s32 cpu, u32 my_llc_id)
{
struct layer *layer;
u64 dsq_id;
@ -1497,29 +1517,60 @@ __weak int consume_non_open(struct cost *costc, s32 cpu, u32 my_llc_id)
if (!costc)
return -EINVAL;
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
struct cpumask *layer_cpumask;
if (!(layer_cpumask = lookup_layer_cpumask(layer_idx)))
return -ENOENT;
if (!bpf_cpumask_test_cpu(cpu, layer_cpumask) &&
(cpu > nr_possible_cpus || cpu != fallback_cpu || layer->nr_cpus != 0))
continue;
if (local_llc_iteration) {
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
if (scx_bpf_consume(dsq_id))
return 0;
struct cpumask *layer_cpumask;
if (!(layer_cpumask = lookup_layer_cpumask(layer_idx)))
return -ENOENT;
if (!bpf_cpumask_test_cpu(cpu, layer_cpumask) &&
(cpu > nr_possible_cpus ||
cpu != fallback_cpu ||
layer->nr_cpus != 0))
continue;
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
} else {
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
struct cpumask *layer_cpumask;
if (!(layer_cpumask = lookup_layer_cpumask(layer_idx)))
return -ENOENT;
if (!bpf_cpumask_test_cpu(cpu, layer_cpumask) &&
(cpu > nr_possible_cpus || cpu != fallback_cpu ||
layer->nr_cpus != 0))
continue;
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
}
@ -1535,23 +1586,45 @@ __weak int consume_open_no_preempt(struct cost *costc, u32 my_llc_id)
if (!costc)
return -EINVAL;
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
if (layer->preempt || layer->kind == LAYER_KIND_CONFINED)
continue;
if (local_llc_iteration) {
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
if (layer->preempt || layer->kind == LAYER_KIND_CONFINED)
continue;
if (scx_bpf_consume(dsq_id))
return 0;
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
} else {
bpf_for(idx, 0, nr_layers) {
layer_idx = rotate_layer_id(costc->pref_layer, idx);
if (layer_idx >= nr_layers) {
scx_bpf_error("can't happen");
return -EINVAL;
}
layer = MEMBER_VPTR(layers, [layer_idx]);
if (has_budget(costc, layer) == 0)
continue;
if (layer->preempt || layer->kind == LAYER_KIND_CONFINED)
continue;
bpf_for(llc_idx, 0, nr_llcs) {
u32 llc_id = rotate_llc_id(my_llc_id, llc_idx);
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (scx_bpf_consume(dsq_id))
return 0;
}
}
}

5
scheds/rust/scx_layered/src/main.rs
View File

@ -458,6 +458,10 @@ struct Opts {
#[clap(long)]
run_example: bool,
/// Enables iteration over local LLCs first for dispatch.
#[clap(long, default_value = "false")]
local_llc_iteration: bool,
/// Disable antistall
#[clap(long, default_value = "false")]
disable_antistall: bool,
@ -1494,6 +1498,7 @@ impl<'a> Scheduler<'a> {
skel.maps.rodata_data.has_little_cores = topo.has_little_cores();
skel.maps.rodata_data.disable_topology = disable_topology;
skel.maps.rodata_data.xnuma_preemption = opts.xnuma_preemption;
skel.maps.rodata_data.local_llc_iteration = opts.local_llc_iteration;
skel.maps.rodata_data.antistall_sec = opts.antistall_sec;
if opts.monitor_disable {
skel.maps.rodata_data.monitor_disable = opts.monitor_disable;