mirror of
https://github.com/JakeHillion/scx.git
synced 2024-11-25 11:00:24 +00:00
set all enums in userspace before loading
This commit is contained in:
Emil Tsalapatis
parent
ff861d3e2c
commit
31b9fb4135
@ -58,8 +58,8 @@ enum {
|
||||
|
||||
const volatile s32 central_cpu;
|
||||
const volatile u32 nr_cpu_ids = 1; /* !0 for veristat, set during init */
|
||||
const volatile u64 slice_ns;
|
||||
|
||||
u64 slice_ns = 0;
|
||||
bool timer_pinned = true;
|
||||
u64 nr_total, nr_locals, nr_queued, nr_lost_pids;
|
||||
u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries;
|
||||
@ -121,21 +121,21 @@ void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags)
|
||||
*/
|
||||
if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {
|
||||
__sync_fetch_and_add(&nr_locals, 1);
|
||||
scx_bpf_dispatch(p, SCX_ENUM(DSQ_LOCAL), SCX_ENUM(SLICE_INF),
|
||||
enq_flags | SCX_ENUM(ENQ_PREEMPT));
|
||||
scx_bpf_dispatch(p, SCX_ENUM(SCX_DSQ_LOCAL), SCX_ENUM(SCX_SLICE_INF),
|
||||
enq_flags | SCX_ENUM(SCX_ENQ_PREEMPT));
|
||||
return;
|
||||
}
|
||||
|
||||
if (bpf_map_push_elem(¢ral_q, &pid, 0)) {
|
||||
__sync_fetch_and_add(&nr_overflows, 1);
|
||||
scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_ENUM(SLICE_INF), enq_flags);
|
||||
scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_ENUM(SCX_SLICE_INF), enq_flags);
|
||||
return;
|
||||
}
|
||||
|
||||
__sync_fetch_and_add(&nr_queued, 1);
|
||||
|
||||
if (!scx_bpf_task_running(p))
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(KICK_PREEMPT));
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(SCX_KICK_PREEMPT));
|
||||
}
|
||||
|
||||
static bool dispatch_to_cpu(s32 cpu)
|
||||
@ -161,7 +161,7 @@ static bool dispatch_to_cpu(s32 cpu)
|
||||
*/
|
||||
if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) {
|
||||
__sync_fetch_and_add(&nr_mismatches, 1);
|
||||
scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_ENUM(SLICE_INF), 0);
|
||||
scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_ENUM(SCX_SLICE_INF), 0);
|
||||
bpf_task_release(p);
|
||||
/*
|
||||
* We might run out of dispatch buffer slots if we continue dispatching
|
||||
@ -175,10 +175,10 @@ static bool dispatch_to_cpu(s32 cpu)
|
||||
}
|
||||
|
||||
/* dispatch to local and mark that @cpu doesn't need more */
|
||||
scx_bpf_dispatch(p, SCX_ENUM(DSQ_LOCAL_ON) | cpu, SCX_ENUM(SLICE_INF), 0);
|
||||
scx_bpf_dispatch(p, SCX_ENUM(SCX_DSQ_LOCAL_ON) | cpu, SCX_ENUM(SCX_SLICE_INF), 0);
|
||||
|
||||
if (cpu != central_cpu)
|
||||
scx_bpf_kick_cpu(cpu, SCX_ENUM(KICK_IDLE));
|
||||
scx_bpf_kick_cpu(cpu, SCX_ENUM(SCX_KICK_IDLE));
|
||||
|
||||
bpf_task_release(p);
|
||||
return true;
|
||||
@ -217,7 +217,7 @@ void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev)
|
||||
*/
|
||||
if (!scx_bpf_dispatch_nr_slots()) {
|
||||
__sync_fetch_and_add(&nr_retries, 1);
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(KICK_PREEMPT));
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(SCX_KICK_PREEMPT));
|
||||
return;
|
||||
}
|
||||
|
||||
@ -239,7 +239,7 @@ void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev)
|
||||
* Force dispatch on the scheduling CPU so that it finds a task
|
||||
* to run for us.
|
||||
*/
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(KICK_PREEMPT));
|
||||
scx_bpf_kick_cpu(central_cpu, SCX_ENUM(SCX_KICK_PREEMPT));
|
||||
}
|
||||
}
|
||||
|
||||
@ -282,19 +282,19 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer)
|
||||
/* kick iff the current one exhausted its slice */
|
||||
started_at = ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids);
|
||||
if (started_at && *started_at &&
|
||||
vtime_before(now, *started_at + SCX_ENUM(SLICE_DFL)))
|
||||
vtime_before(now, *started_at + SCX_ENUM(SCX_SLICE_DFL)))
|
||||
continue;
|
||||
|
||||
/* and there's something pending */
|
||||
if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) ||
|
||||
scx_bpf_dsq_nr_queued(SCX_ENUM(DSQ_LOCAL_ON) | cpu))
|
||||
scx_bpf_dsq_nr_queued(SCX_ENUM(SCX_DSQ_LOCAL_ON) | cpu))
|
||||
;
|
||||
else if (nr_to_kick)
|
||||
nr_to_kick--;
|
||||
else
|
||||
continue;
|
||||
|
||||
scx_bpf_kick_cpu(cpu, SCX_ENUM(KICK_PREEMPT));
|
||||
scx_bpf_kick_cpu(cpu, SCX_ENUM(SCX_KICK_PREEMPT));
|
||||
}
|
||||
|
||||
bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN);
|
||||
@ -308,14 +308,6 @@ int BPF_STRUCT_OPS_SLEEPABLE(central_init)
|
||||
struct bpf_timer *timer;
|
||||
int ret;
|
||||
|
||||
ret = SCX_ENUM_INIT();
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/* Use the default slice size if the user didn't set it themselves. */
|
||||
if (slice_ns == 0)
|
||||
slice_ns = SCX_ENUM(SLICE_DFL);
|
||||
|
||||
ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
@ -56,14 +56,16 @@ int main(int argc, char **argv)
|
||||
signal(SIGTERM, sigint_handler);
|
||||
restart:
|
||||
skel = SCX_OPS_OPEN(central_ops, scx_central);
|
||||
SCX_ENUM_INIT(skel);
|
||||
|
||||
skel->rodata->central_cpu = 0;
|
||||
skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
|
||||
skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
|
||||
|
||||
while ((opt = getopt(argc, argv, "s:c:pvh")) != -1) {
|
||||
switch (opt) {
|
||||
case 's':
|
||||
skel->bss->slice_ns = strtoull(optarg, NULL, 0) * 1000;
|
||||
skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
|
||||
break;
|
||||
case 'c':
|
||||
skel->rodata->central_cpu = strtoul(optarg, NULL, 0);
|
||||
|
||||
@ -9,67 +9,47 @@
|
||||
|
||||
#include <scx/enums.h>
|
||||
|
||||
#define SCX_ENUM_DEFINE() \
|
||||
struct scx_enums _scx_enum;
|
||||
#define SCX_ENUM_DEFINE(name) const volatile u64 SCX_ENUM(name);
|
||||
|
||||
#define SCX_ENUM(name) \
|
||||
(_scx_enum.SCX_##name)
|
||||
SCX_ENUM_DEFINE(SCX_OPS_NAME_LEN);
|
||||
SCX_ENUM_DEFINE(SCX_SLICE_DFL);
|
||||
SCX_ENUM_DEFINE(SCX_SLICE_INF);
|
||||
|
||||
#define SCX_ENUM_SET(enums, etype, name) \
|
||||
do { \
|
||||
if (!bpf_core_enum_value_exists(enum etype, name)) { \
|
||||
scx_bpf_error("missing enum %s\n", #name); \
|
||||
return -EINVAL; \
|
||||
} \
|
||||
(enums)->name = bpf_core_enum_value(enum etype, name); \
|
||||
} while (0)
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_FLAG_BUILTIN);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_FLAG_LOCAL_ON);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_INVALID);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_GLOBAL);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_LOCAL);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_LOCAL_ON);
|
||||
SCX_ENUM_DEFINE(SCX_DSQ_LOCAL_CPU_MASK);
|
||||
|
||||
static inline int
|
||||
scx_enum_init(struct scx_enums *enums)
|
||||
{
|
||||
SCX_ENUM_SET(enums, scx_public_consts, SCX_OPS_NAME_LEN);
|
||||
SCX_ENUM_SET(enums, scx_public_consts, SCX_SLICE_DFL);
|
||||
SCX_ENUM_SET(enums, scx_public_consts, SCX_SLICE_INF);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_QUEUED);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_RESET_RUNNABLE_AT);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_DEQD_FOR_SLEEP);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_STATE_SHIFT);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_STATE_BITS);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_STATE_MASK);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_CURSOR);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_INVALID);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_GLOBAL);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_LOCAL);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_LOCAL_ON);
|
||||
SCX_ENUM_SET(enums, scx_dsq_id_flags, SCX_DSQ_LOCAL_CPU_MASK);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_NONE);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_INIT);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_READY);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_ENABLED);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_NR_STATES);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_QUEUED);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_RESET_RUNNABLE_AT);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_DEQD_FOR_SLEEP);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_STATE_SHIFT);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_STATE_BITS);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_STATE_MASK);
|
||||
SCX_ENUM_SET(enums, scx_ent_flags, SCX_TASK_CURSOR);
|
||||
SCX_ENUM_DEFINE(SCX_TASK_DSQ_ON_PRIQ);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_task_state, SCX_TASK_NONE);
|
||||
SCX_ENUM_SET(enums, scx_task_state, SCX_TASK_INIT);
|
||||
SCX_ENUM_SET(enums, scx_task_state, SCX_TASK_READY);
|
||||
SCX_ENUM_SET(enums, scx_task_state, SCX_TASK_ENABLED);
|
||||
SCX_ENUM_SET(enums, scx_task_state, SCX_TASK_NR_STATES);
|
||||
SCX_ENUM_DEFINE(SCX_KICK_IDLE);
|
||||
SCX_ENUM_DEFINE(SCX_KICK_PREEMPT);
|
||||
SCX_ENUM_DEFINE(SCX_KICK_WAIT);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_kick_flags, SCX_KICK_IDLE);
|
||||
SCX_ENUM_SET(enums, scx_kick_flags, SCX_KICK_PREEMPT);
|
||||
SCX_ENUM_SET(enums, scx_kick_flags, SCX_KICK_WAIT);
|
||||
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_WAKEUP);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_HEAD);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_PREEMPT);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_REENQ);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_LAST);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_CLEAR_OPSS);
|
||||
SCX_ENUM_SET(enums, scx_enq_flags, SCX_ENQ_DSQ_PRIQ);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define SCX_ENUM_INIT() scx_enum_init(&_scx_enum)
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_WAKEUP);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_HEAD);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_CPU_SELECTED);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_PREEMPT);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_REENQ);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_LAST);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_CLEAR_OPSS);
|
||||
SCX_ENUM_DEFINE(SCX_ENQ_DSQ_PRIQ);
|
||||
|
||||
#endif /* __SCX_ENUMS_BPF_H */
|
||||
|
||||
@ -9,47 +9,63 @@
|
||||
#ifndef __SCX_ENUMS_H
|
||||
#define __SCX_ENUMS_H
|
||||
|
||||
struct scx_enums {
|
||||
u64 SCX_OPS_NAME_LEN;
|
||||
u64 SCX_SLICE_DFL;
|
||||
u64 SCX_SLICE_INF;
|
||||
#define SCX_ENUM(name) __##name
|
||||
|
||||
u64 SCX_DSQ_FLAG_BUILTIN;
|
||||
u64 SCX_DSQ_FLAG_LOCAL_ON;
|
||||
u64 SCX_DSQ_INVALID;
|
||||
u64 SCX_DSQ_GLOBAL;
|
||||
u64 SCX_DSQ_LOCAL;
|
||||
u64 SCX_DSQ_LOCAL_ON;
|
||||
u64 SCX_DSQ_LOCAL_CPU_MASK;
|
||||
#ifndef __bpf__
|
||||
|
||||
u64 SCX_TASK_QUEUED;
|
||||
u64 SCX_TASK_RESET_RUNNABLE_AT;
|
||||
u64 SCX_TASK_DEQD_FOR_SLEEP;
|
||||
u64 SCX_TASK_STATE_SHIFT;
|
||||
u64 SCX_TASK_STATE_BITS;
|
||||
u64 SCX_TASK_STATE_MASK;
|
||||
u64 SCX_TASK_CURSOR;
|
||||
static inline void __ENUM_set(u64 *val, char *type, char *name)
|
||||
{
|
||||
bool res;
|
||||
|
||||
u64 SCX_TASK_NONE;
|
||||
u64 SCX_TASK_INIT;
|
||||
u64 SCX_TASK_READY;
|
||||
u64 SCX_TASK_ENABLED;
|
||||
u64 SCX_TASK_NR_STATES;
|
||||
res = __COMPAT_read_enum(type, name, val);
|
||||
SCX_BUG_ON(!res, "enum not found(%s)", name);
|
||||
}
|
||||
|
||||
u64 SCX_TASK_DSQ_ON_PRIQ;
|
||||
#define SCX_ENUM_SET(skel, type, name) do { \
|
||||
__ENUM_set(&skel->rodata->SCX_ENUM(name), #type, #name); \
|
||||
} while (0)
|
||||
|
||||
u64 SCX_KICK_IDLE;
|
||||
u64 SCX_KICK_PREEMPT;
|
||||
u64 SCX_KICK_WAIT;
|
||||
#define SCX_ENUM_INIT(skel) \
|
||||
SCX_ENUM_SET(skel, scx_public_consts, SCX_OPS_NAME_LEN); \
|
||||
SCX_ENUM_SET(skel, scx_public_consts, SCX_SLICE_DFL); \
|
||||
SCX_ENUM_SET(skel, scx_public_consts, SCX_SLICE_INF); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_INVALID); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_GLOBAL); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL_ON); \
|
||||
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL_CPU_MASK); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_QUEUED); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_RESET_RUNNABLE_AT); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_DEQD_FOR_SLEEP); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_SHIFT); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \
|
||||
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \
|
||||
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \
|
||||
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \
|
||||
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \
|
||||
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \
|
||||
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \
|
||||
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \
|
||||
\
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \
|
||||
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_DSQ_PRIQ);
|
||||
|
||||
u64 SCX_ENQ_WAKEUP;
|
||||
u64 SCX_ENQ_HEAD;
|
||||
u64 SCX_ENQ_CPU_SELECTED;
|
||||
u64 SCX_ENQ_PREEMPT;
|
||||
u64 SCX_ENQ_REENQ;
|
||||
u64 SCX_ENQ_LAST;
|
||||
u64 SCX_ENQ_CLEAR_OPSS;
|
||||
u64 SCX_ENQ_DSQ_PRIQ;
|
||||
};
|
||||
#endif /* !__bpf__ */
|
||||
|
||||
#endif /* __SCX_ENUMS_H */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user