mirror of
https://github.com/sched-ext/scx.git
synced 2024-11-21 18:41:47 +00:00
Merge branch 'main' into scx-fair
This commit is contained in:
Andrea Righi
GitHub
commit
5b4b6df5e4
@ -2631,7 +2631,8 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(layered_init)
|
||||
bpf_for(i, 0, nr_possible_cpus) {
|
||||
const volatile u8 *u8_ptr;
|
||||
|
||||
init_antistall_dsq = bpf_map_lookup_percpu_elem(&antistall_cpu_dsq, &zero, i);
|
||||
init_antistall_dsq = bpf_map_lookup_percpu_elem(&antistall_cpu_dsq,
|
||||
&zero, i);
|
||||
if (init_antistall_dsq) {
|
||||
*init_antistall_dsq = SCX_DSQ_INVALID;
|
||||
}
|
||||
|
||||
@ -41,20 +41,20 @@ impl LayerSpec {
|
||||
Ok(config.specs)
|
||||
}
|
||||
|
||||
pub fn nodes(&self) -> Vec<usize> {
|
||||
match &self.kind {
|
||||
LayerKind::Confined { nodes, .. }
|
||||
| LayerKind::Open { nodes, .. }
|
||||
| LayerKind::Grouped { nodes, .. } => nodes.clone(),
|
||||
}
|
||||
pub fn nodes(&self) -> &Vec<usize> {
|
||||
&self.kind.common().nodes
|
||||
}
|
||||
|
||||
pub fn llcs(&self) -> Vec<usize> {
|
||||
match &self.kind {
|
||||
LayerKind::Confined { llcs, .. }
|
||||
| LayerKind::Open { llcs, .. }
|
||||
| LayerKind::Grouped { llcs, .. } => llcs.clone(),
|
||||
pub fn llcs(&self) -> &Vec<usize> {
|
||||
&self.kind.common().llcs
|
||||
}
|
||||
|
||||
pub fn nodes_mut(&mut self) -> &mut Vec<usize> {
|
||||
&mut self.kind.common_mut().nodes
|
||||
}
|
||||
|
||||
pub fn llcs_mut(&mut self) -> &mut Vec<usize> {
|
||||
&mut self.kind.common_mut().llcs
|
||||
}
|
||||
}
|
||||
|
||||
@ -73,91 +73,55 @@ pub enum LayerMatch {
|
||||
TGIDEquals(u32),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct LayerCommon {
|
||||
#[serde(default)]
|
||||
pub min_exec_us: u64,
|
||||
#[serde(default)]
|
||||
pub yield_ignore: f64,
|
||||
#[serde(default)]
|
||||
pub slice_us: u64,
|
||||
#[serde(default)]
|
||||
pub preempt: bool,
|
||||
#[serde(default)]
|
||||
pub preempt_first: bool,
|
||||
#[serde(default)]
|
||||
pub exclusive: bool,
|
||||
#[serde(default)]
|
||||
pub weight: u32,
|
||||
#[serde(default)]
|
||||
pub idle_smt: bool,
|
||||
#[serde(default)]
|
||||
pub growth_algo: LayerGrowthAlgo,
|
||||
#[serde(default)]
|
||||
pub perf: u64,
|
||||
#[serde(default)]
|
||||
pub nodes: Vec<usize>,
|
||||
#[serde(default)]
|
||||
pub llcs: Vec<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum LayerKind {
|
||||
Confined {
|
||||
util_range: (f64, f64),
|
||||
#[serde(default)]
|
||||
cpus_range: Option<(usize, usize)>,
|
||||
#[serde(default)]
|
||||
min_exec_us: u64,
|
||||
#[serde(default)]
|
||||
yield_ignore: f64,
|
||||
#[serde(default)]
|
||||
slice_us: u64,
|
||||
#[serde(default)]
|
||||
preempt: bool,
|
||||
#[serde(default)]
|
||||
preempt_first: bool,
|
||||
#[serde(default)]
|
||||
exclusive: bool,
|
||||
#[serde(default)]
|
||||
weight: u32,
|
||||
#[serde(default)]
|
||||
idle_smt: bool,
|
||||
#[serde(default)]
|
||||
growth_algo: LayerGrowthAlgo,
|
||||
#[serde(default)]
|
||||
perf: u64,
|
||||
#[serde(default)]
|
||||
nodes: Vec<usize>,
|
||||
#[serde(default)]
|
||||
llcs: Vec<usize>,
|
||||
|
||||
#[serde(flatten)]
|
||||
common: LayerCommon,
|
||||
},
|
||||
Grouped {
|
||||
util_range: (f64, f64),
|
||||
#[serde(default)]
|
||||
cpus_range: Option<(usize, usize)>,
|
||||
#[serde(default)]
|
||||
min_exec_us: u64,
|
||||
#[serde(default)]
|
||||
yield_ignore: f64,
|
||||
#[serde(default)]
|
||||
slice_us: u64,
|
||||
#[serde(default)]
|
||||
preempt: bool,
|
||||
#[serde(default)]
|
||||
preempt_first: bool,
|
||||
#[serde(default)]
|
||||
exclusive: bool,
|
||||
#[serde(default)]
|
||||
weight: u32,
|
||||
#[serde(default)]
|
||||
idle_smt: bool,
|
||||
#[serde(default)]
|
||||
growth_algo: LayerGrowthAlgo,
|
||||
#[serde(default)]
|
||||
perf: u64,
|
||||
#[serde(default)]
|
||||
nodes: Vec<usize>,
|
||||
#[serde(default)]
|
||||
llcs: Vec<usize>,
|
||||
|
||||
#[serde(flatten)]
|
||||
common: LayerCommon,
|
||||
},
|
||||
Open {
|
||||
#[serde(default)]
|
||||
min_exec_us: u64,
|
||||
#[serde(default)]
|
||||
yield_ignore: f64,
|
||||
#[serde(default)]
|
||||
slice_us: u64,
|
||||
#[serde(default)]
|
||||
preempt: bool,
|
||||
#[serde(default)]
|
||||
preempt_first: bool,
|
||||
#[serde(default)]
|
||||
exclusive: bool,
|
||||
#[serde(default)]
|
||||
weight: u32,
|
||||
#[serde(default)]
|
||||
idle_smt: bool,
|
||||
#[serde(default)]
|
||||
growth_algo: LayerGrowthAlgo,
|
||||
#[serde(default)]
|
||||
perf: u64,
|
||||
#[serde(default)]
|
||||
nodes: Vec<usize>,
|
||||
#[serde(default)]
|
||||
llcs: Vec<usize>,
|
||||
#[serde(flatten)]
|
||||
common: LayerCommon,
|
||||
},
|
||||
}
|
||||
|
||||
@ -169,4 +133,20 @@ impl LayerKind {
|
||||
LayerKind::Open { .. } => bpf_intf::layer_kind_LAYER_KIND_OPEN as i32,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn common(&self) -> &LayerCommon {
|
||||
match self {
|
||||
LayerKind::Confined { common, .. }
|
||||
| LayerKind::Grouped { common, .. }
|
||||
| LayerKind::Open { common, .. } => common,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn common_mut(&mut self) -> &mut LayerCommon {
|
||||
match self {
|
||||
LayerKind::Confined { common, .. }
|
||||
| LayerKind::Grouped { common, .. }
|
||||
| LayerKind::Open { common, .. } => common,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ use std::collections::BTreeMap;
|
||||
use anyhow::bail;
|
||||
use anyhow::Result;
|
||||
use bitvec::prelude::*;
|
||||
pub use config::LayerCommon;
|
||||
pub use config::LayerConfig;
|
||||
pub use config::LayerKind;
|
||||
pub use config::LayerMatch;
|
||||
|
||||
@ -28,6 +28,7 @@ use bitvec::prelude::*;
|
||||
pub use bpf_skel::*;
|
||||
use clap::Parser;
|
||||
use crossbeam::channel::RecvTimeoutError;
|
||||
use lazy_static::lazy_static;
|
||||
use libbpf_rs::skel::OpenSkel;
|
||||
use libbpf_rs::skel::Skel;
|
||||
use libbpf_rs::skel::SkelBuilder;
|
||||
@ -74,12 +75,10 @@ const NR_LSTATS: usize = bpf_intf::layer_stat_idx_NR_LSTATS as usize;
|
||||
const NR_LAYER_MATCH_KINDS: usize = bpf_intf::layer_match_kind_NR_LAYER_MATCH_KINDS as usize;
|
||||
const MAX_LAYER_NAME: usize = bpf_intf::consts_MAX_LAYER_NAME as usize;
|
||||
|
||||
#[rustfmt::skip]
|
||||
lazy_static::lazy_static! {
|
||||
lazy_static! {
|
||||
static ref NR_POSSIBLE_CPUS: usize = libbpf_rs::num_possible_cpus().unwrap();
|
||||
static ref USAGE_DECAY: f64 = 0.5f64.powf(1.0 / USAGE_HALF_LIFE_F64);
|
||||
static ref EXAMPLE_CONFIG: LayerConfig =
|
||||
LayerConfig {
|
||||
static ref EXAMPLE_CONFIG: LayerConfig = LayerConfig {
|
||||
specs: vec![
|
||||
LayerSpec {
|
||||
name: "batch".into(),
|
||||
@ -89,8 +88,9 @@ lazy_static::lazy_static! {
|
||||
vec![LayerMatch::NiceAbove(0)],
|
||||
],
|
||||
kind: LayerKind::Confined {
|
||||
cpus_range: Some((0, 16)),
|
||||
util_range: (0.8, 0.9),
|
||||
cpus_range: Some((0, 16)),
|
||||
common: LayerCommon {
|
||||
min_exec_us: 1000,
|
||||
yield_ignore: 0.0,
|
||||
preempt: false,
|
||||
@ -105,6 +105,7 @@ lazy_static::lazy_static! {
|
||||
llcs: vec![],
|
||||
},
|
||||
},
|
||||
},
|
||||
LayerSpec {
|
||||
name: "immediate".into(),
|
||||
comment: Some("tasks under workload.slice with nice value < 0".into()),
|
||||
@ -113,6 +114,7 @@ lazy_static::lazy_static! {
|
||||
LayerMatch::NiceBelow(0),
|
||||
]],
|
||||
kind: LayerKind::Open {
|
||||
common: LayerCommon {
|
||||
min_exec_us: 100,
|
||||
yield_ignore: 0.25,
|
||||
preempt: true,
|
||||
@ -127,20 +129,20 @@ lazy_static::lazy_static! {
|
||||
llcs: vec![],
|
||||
},
|
||||
},
|
||||
},
|
||||
LayerSpec {
|
||||
name: "stress-ng".into(),
|
||||
comment: Some("stress-ng test layer".into()),
|
||||
matches: vec![vec![
|
||||
LayerMatch::CommPrefix("stress-ng".into()),
|
||||
matches: vec![
|
||||
vec![LayerMatch::CommPrefix("stress-ng".into()),],
|
||||
vec![LayerMatch::PcommPrefix("stress-ng".into()),]
|
||||
],
|
||||
vec![
|
||||
LayerMatch::PcommPrefix("stress-ng".into()),
|
||||
]],
|
||||
kind: LayerKind::Confined {
|
||||
cpus_range: None,
|
||||
util_range: (0.2, 0.8),
|
||||
common: LayerCommon {
|
||||
min_exec_us: 800,
|
||||
yield_ignore: 0.0,
|
||||
util_range: (0.2, 0.8),
|
||||
preempt: true,
|
||||
preempt_first: false,
|
||||
exclusive: false,
|
||||
@ -153,6 +155,7 @@ lazy_static::lazy_static! {
|
||||
llcs: vec![],
|
||||
},
|
||||
},
|
||||
},
|
||||
LayerSpec {
|
||||
name: "normal".into(),
|
||||
comment: Some("the rest".into()),
|
||||
@ -160,6 +163,7 @@ lazy_static::lazy_static! {
|
||||
kind: LayerKind::Grouped {
|
||||
cpus_range: None,
|
||||
util_range: (0.5, 0.6),
|
||||
common: LayerCommon {
|
||||
min_exec_us: 200,
|
||||
yield_ignore: 0.0,
|
||||
preempt: false,
|
||||
@ -174,6 +178,7 @@ lazy_static::lazy_static! {
|
||||
llcs: vec![],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
@ -923,8 +928,7 @@ impl Layer {
|
||||
LayerKind::Confined {
|
||||
cpus_range,
|
||||
util_range,
|
||||
nodes,
|
||||
llcs,
|
||||
common: LayerCommon { nodes, llcs, .. },
|
||||
..
|
||||
} => {
|
||||
let cpus_range = cpus_range.unwrap_or((0, std::usize::MAX));
|
||||
@ -962,7 +966,14 @@ impl Layer {
|
||||
bail!("invalid util_range {:?}", util_range);
|
||||
}
|
||||
}
|
||||
LayerKind::Grouped { nodes, llcs, .. } | LayerKind::Open { nodes, llcs, .. } => {
|
||||
LayerKind::Grouped {
|
||||
common: LayerCommon { nodes, llcs, .. },
|
||||
..
|
||||
}
|
||||
| LayerKind::Open {
|
||||
common: LayerCommon { nodes, llcs, .. },
|
||||
..
|
||||
} => {
|
||||
if nodes.len() == 0 && llcs.len() == 0 {
|
||||
allowed_cpus.fill(true);
|
||||
} else {
|
||||
@ -987,23 +998,13 @@ impl Layer {
|
||||
}
|
||||
}
|
||||
|
||||
let layer_growth_algo = match &kind {
|
||||
LayerKind::Confined { growth_algo, .. }
|
||||
| LayerKind::Grouped { growth_algo, .. }
|
||||
| LayerKind::Open { growth_algo, .. } => growth_algo.clone(),
|
||||
};
|
||||
let preempt = match &kind {
|
||||
LayerKind::Confined { preempt, .. }
|
||||
| LayerKind::Grouped { preempt, .. }
|
||||
| LayerKind::Open { preempt, .. } => preempt.clone(),
|
||||
};
|
||||
let layer_growth_algo = kind.common().growth_algo.clone();
|
||||
let preempt = kind.common().preempt;
|
||||
|
||||
let core_order = layer_growth_algo.layer_core_order(cpu_pool, spec, idx, topo);
|
||||
debug!(
|
||||
"layer: {} algo: {:?} core order: {:?}",
|
||||
name,
|
||||
layer_growth_algo.clone(),
|
||||
core_order
|
||||
name, &layer_growth_algo, core_order
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
@ -1289,8 +1290,8 @@ impl<'a> Scheduler<'a> {
|
||||
layer.nr_match_ors = spec.matches.len() as u32;
|
||||
layer.kind = spec.kind.as_bpf_enum();
|
||||
|
||||
match &spec.kind {
|
||||
LayerKind::Confined {
|
||||
{
|
||||
let LayerCommon {
|
||||
min_exec_us,
|
||||
yield_ignore,
|
||||
perf,
|
||||
@ -1303,35 +1304,8 @@ impl<'a> Scheduler<'a> {
|
||||
slice_us,
|
||||
weight,
|
||||
..
|
||||
}
|
||||
| LayerKind::Grouped {
|
||||
min_exec_us,
|
||||
yield_ignore,
|
||||
perf,
|
||||
preempt,
|
||||
preempt_first,
|
||||
exclusive,
|
||||
idle_smt,
|
||||
growth_algo,
|
||||
nodes,
|
||||
slice_us,
|
||||
weight,
|
||||
..
|
||||
}
|
||||
| LayerKind::Open {
|
||||
min_exec_us,
|
||||
yield_ignore,
|
||||
perf,
|
||||
preempt,
|
||||
preempt_first,
|
||||
exclusive,
|
||||
idle_smt,
|
||||
growth_algo,
|
||||
nodes,
|
||||
slice_us,
|
||||
weight,
|
||||
..
|
||||
} => {
|
||||
} = spec.kind.common();
|
||||
|
||||
layer.slice_ns = if *slice_us > 0 {
|
||||
*slice_us * 1000
|
||||
} else {
|
||||
@ -1368,7 +1342,6 @@ impl<'a> Scheduler<'a> {
|
||||
layer.cache_mask |= cachemask_from_llcs(&topo_node.llcs()) as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
perf_set |= layer.perf > 0;
|
||||
}
|
||||
@ -1449,14 +1422,8 @@ impl<'a> Scheduler<'a> {
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.map(|mut s| {
|
||||
match &mut s.kind {
|
||||
LayerKind::Confined { nodes, llcs, .. }
|
||||
| LayerKind::Open { nodes, llcs, .. }
|
||||
| LayerKind::Grouped { nodes, llcs, .. } => {
|
||||
nodes.truncate(0);
|
||||
llcs.truncate(0);
|
||||
}
|
||||
};
|
||||
s.kind.common_mut().nodes.clear();
|
||||
s.kind.common_mut().llcs.clear();
|
||||
s
|
||||
})
|
||||
.collect()
|
||||
|
||||
@ -25,7 +25,6 @@ use serde::Serialize;
|
||||
use crate::bpf_intf;
|
||||
use crate::BpfStats;
|
||||
use crate::Layer;
|
||||
use crate::LayerKind;
|
||||
use crate::Stats;
|
||||
|
||||
fn fmt_pct(v: f64) -> String {
|
||||
@ -174,12 +173,6 @@ impl LayerStats {
|
||||
if b != 0.0 { a / b * 100.0 } else { 0.0 }
|
||||
};
|
||||
|
||||
let is_excl = match &layer.kind {
|
||||
LayerKind::Confined { exclusive, .. }
|
||||
| LayerKind::Grouped { exclusive, .. }
|
||||
| LayerKind::Open { exclusive, .. } => *exclusive,
|
||||
} as u32;
|
||||
|
||||
Self {
|
||||
util: stats.layer_utils[lidx] * 100.0,
|
||||
util_frac: calc_frac(stats.layer_utils[lidx], stats.total_util),
|
||||
@ -206,7 +199,7 @@ impl LayerStats {
|
||||
keep: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_KEEP),
|
||||
keep_fail_max_exec: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_KEEP_FAIL_MAX_EXEC),
|
||||
keep_fail_busy: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_KEEP_FAIL_BUSY),
|
||||
is_excl,
|
||||
is_excl: layer.kind.common().exclusive as u32,
|
||||
excl_collision: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_EXCL_COLLISION),
|
||||
excl_preempt: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_EXCL_PREEMPT),
|
||||
kick: lstat_pct(bpf_intf::layer_stat_idx_LSTAT_KICK),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user