From 0a518e9f9e70a63f687fd7b0ffa4f6c9ec47ac10 Mon Sep 17 00:00:00 2001 From: Daniel Hodges Date: Fri, 1 Nov 2024 04:44:12 -0700 Subject: [PATCH] scx_layered: Add additional drain to fallback DSQs Fallback DSQs are not accounted with costs. If a layer is saturating the machine it is possible to not consume from the fallback DSQ and stall the task. This introduces and additional consumption from the fallback DSQ when a layer runs out of budget. In addition, tasks that use partial CPU affinities should be placed into the fallback DSQ. This change was tested with stress-ng --cacheline `nproc` for several minutes without causing stalls (which would stall on main). Signed-off-by: Daniel Hodges --- scheds/rust/scx_layered/src/bpf/cost.bpf.c | 2 ++ scheds/rust/scx_layered/src/bpf/main.bpf.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scheds/rust/scx_layered/src/bpf/cost.bpf.c b/scheds/rust/scx_layered/src/bpf/cost.bpf.c index e949164..4c15f6e 100644 --- a/scheds/rust/scx_layered/src/bpf/cost.bpf.c +++ b/scheds/rust/scx_layered/src/bpf/cost.bpf.c @@ -15,6 +15,7 @@ struct cost { u32 idx; bool overflow; bool has_parent; + bool drain_fallback; }; @@ -217,6 +218,7 @@ static int record_cpu_cost(struct cost *costc, u32 layer_id, s64 amount) __sync_fetch_and_sub(&costc->budget[layer_id], amount); if (costc->budget[layer_id] <= 0) { + costc->drain_fallback = true; if (costc->has_parent) { s64 budget = acquire_budget(costc, layer_id, costc->capacity[layer_id] + amount); diff --git a/scheds/rust/scx_layered/src/bpf/main.bpf.c b/scheds/rust/scx_layered/src/bpf/main.bpf.c index 388b0f2..624340a 100644 --- a/scheds/rust/scx_layered/src/bpf/main.bpf.c +++ b/scheds/rust/scx_layered/src/bpf/main.bpf.c @@ -1067,7 +1067,7 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags) * usually important for system performance and responsiveness. */ if (!layer->preempt && - (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { + (p->flags & PF_KTHREAD) && p->nr_cpus_allowed < nr_possible_cpus) { struct cpumask *layer_cpumask; if (!layer->open && @@ -1444,6 +1444,18 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev) return; } + /* + * Fallback DSQs don't have cost accounting. When the budget runs out + * for a layer we do an extra consume of the fallback DSQ to ensure + * that it doesn't stall out when the system is being saturated. + */ + if (costc->drain_fallback) { + costc->drain_fallback = false; + dsq_id = cpu_hi_fallback_dsq_id(cpu); + if (scx_bpf_consume(dsq_id)) + return; + } + u32 my_llc_id = cpu_to_llc_id(cpu); /* consume preempting layers first */