compat: Implement scx_utils::compat and fix up scx_layered

Implement scx_utils::compat to match C's scx/compat.h and update scx_layered. Other rust scheds are still broken.
2024-11-24 20:00:22 +00:00 · 2024-03-28 13:27:32 -10:00 · 2024-03-28 13:27:32 -10:00 · 59bbd800c1
commit 59bbd800c1
parent 891df57b98
14 changed files with 351 additions and 90 deletions
--- a/rust/scx_rustland_core/Cargo.toml
+++ b/rust/scx_rustland_core/Cargo.toml
@ -15,7 +15,7 @@ include = [

 [dependencies]
 anyhow = "1.0"
-libbpf-rs = "0.22.0"
+libbpf-rs = "0.22"
 libc = "0.2.137"
 buddy-alloc = "0.5.1"
 scx_utils = { path = "../scx_utils", version = "0.6" }
@ -24,3 +24,8 @@ scx_utils = { path = "../scx_utils", version = "0.6" }
 tar = "0.4"
 walkdir = "2.4"
 scx_utils = { path = "../scx_utils", version = "0.6" }
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "75042c653ee956b8c262e41ca4bcfcb0e2c461a1" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
--- a/rust/scx_utils/Cargo.toml
+++ b/rust/scx_utils/Cargo.toml
@ -17,9 +17,10 @@ glob = "0.3"
 hex = "0.4.3"
 lazy_static = "1.4"
 libbpf-cargo = "0.22"
-libbpf-rs = "0.22.0"
-buddy-alloc = "0.5.1"
-log = "0.4.17"
+libbpf-rs = "0.22"
+buddy-alloc = "0.5"
+log = "0.4"
+paste = "1.0"
 regex = "1.10"
 sscanf = "0.4"
 tar = "0.4"
@ -30,3 +31,8 @@ version-compare = "0.1"
 bindgen = ">=0.68, <0.70"
 tar = "0.4"
 walkdir = "2.4"
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "125444300e2db452131dd6453101599c1a277784" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "125444300e2db452131dd6453101599c1a277784" }
--- a/rust/scx_utils/src/bpf_builder.rs
+++ b/rust/scx_utils/src/bpf_builder.rs
@ -478,14 +478,6 @@ impl BpfBuilder {
        self
    }

-    fn cflags_string(&self) -> String {
-        self.cflags
-            .iter()
-            .map(|x| x.as_str())
-            .collect::<Vec<&str>>()
-            .join(" ")
-    }
-
    fn bindgen_bpf_intf(&self, deps: &mut BTreeSet<String>) -> Result<()> {
        let (input, output) = match &self.intf_input_output {
            Some(pair) => pair,
@ -538,7 +530,7 @@ impl BpfBuilder {
            .source(input)
            .obj(&obj)
            .clang(&self.clang.0)
-            .clang_args(self.cflags_string())
+            .clang_args(&self.cflags)
            .build_and_generate(&skel_path)?;

        match &self.skel_deps {
--- a/rust/scx_utils/src/builder.rs
+++ b/rust/scx_utils/src/builder.rs
@ -45,6 +45,7 @@ impl Builder {
        let bindings = bindgen::Builder::default()
            .header("bindings.h")
            .allowlist_type("scx_exit_kind")
+            .allowlist_type("scx_internal_consts")
            .parse_callbacks(Box::new(bindgen::CargoCallbacks))
            .generate()
            .expect("Unable to generate bindings");
--- a/rust/scx_utils/src/compat.rs
+++ b/rust/scx_utils/src/compat.rs
@ -0,0 +1,180 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2.
+
+use anyhow::{anyhow, bail, Context, Result};
+use libbpf_rs::libbpf_sys::*;
+use std::ffi::c_void;
+use std::ffi::CStr;
+use std::ffi::CString;
+use std::mem::size_of;
+use std::slice::from_raw_parts;
+
+lazy_static::lazy_static! {
+    pub static ref SCX_OPS_SWITCH_PARTIAL: u64 =
+    read_enum("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL").unwrap_or(0);
+}
+
+fn load_vmlinux_btf() -> &'static mut btf {
+    let btf = unsafe { btf__load_vmlinux_btf() };
+    if btf.is_null() {
+        panic!("btf__load_vmlinux_btf() returned NULL");
+    }
+    unsafe { &mut *btf }
+}
+
+lazy_static::lazy_static! {
+    static ref VMLINUX_BTF: &'static mut btf = load_vmlinux_btf();
+}
+
+fn btf_kind(t: &btf_type) -> u32 {
+    (t.info >> 24) & 0x1f
+}
+
+fn btf_vlen(t: &btf_type) -> u32 {
+    t.info & 0xffff
+}
+
+fn btf_type_plus_1(t: &btf_type) -> *const c_void {
+    let ptr_val = t as *const btf_type as usize;
+    (ptr_val + size_of::<btf_type>()) as *const c_void
+}
+
+fn btf_enum(t: &btf_type) -> &[btf_enum] {
+    let ptr = btf_type_plus_1(t);
+    unsafe { from_raw_parts(ptr as *const btf_enum, btf_vlen(t) as usize) }
+}
+
+fn btf_enum64(t: &btf_type) -> &[btf_enum64] {
+    let ptr = btf_type_plus_1(t);
+    unsafe { from_raw_parts(ptr as *const btf_enum64, btf_vlen(t) as usize) }
+}
+
+fn btf_members(t: &btf_type) -> &[btf_member] {
+    let ptr = btf_type_plus_1(t);
+    unsafe { from_raw_parts(ptr as *const btf_member, btf_vlen(t) as usize) }
+}
+
+fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
+    let n = unsafe { btf__name_by_offset(btf, name_off) };
+    if n.is_null() {
+        bail!("btf__name_by_offset() returned NULL");
+    }
+    Ok(unsafe { CStr::from_ptr(n) }
+        .to_str()
+        .with_context(|| format!("Failed to convert {:?} to string", n))?)
+}
+
+pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
+    let btf: &btf = *VMLINUX_BTF;
+
+    let type_name = CString::new(type_name).unwrap();
+    let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
+    if tid < 0 {
+        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
+    }
+
+    let t = unsafe { btf__type_by_id(btf, tid as _) };
+    if t.is_null() {
+        bail!("btf__type_by_id({}) returned NULL", tid);
+    }
+    let t = unsafe { &*t };
+
+    match btf_kind(t) {
+        BTF_KIND_ENUM => {
+            for e in btf_enum(t).iter() {
+                if btf_name_str_by_offset(btf, e.name_off)? == name {
+                    return Ok(e.val as u64);
+                }
+            }
+        }
+        BTF_KIND_ENUM64 => {
+            for e in btf_enum64(t).iter() {
+                if btf_name_str_by_offset(btf, e.name_off)? == name {
+                    return Ok(((e.val_hi32 as u64) << 32) | (e.val_lo32) as u64);
+                }
+            }
+        }
+        _ => (),
+    }
+
+    Err(anyhow!("{:?} doesn't exist in {:?}", name, type_name))
+}
+
+pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
+    let btf: &btf = *VMLINUX_BTF;
+
+    let type_name = CString::new(type_name).unwrap();
+    let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
+    if tid < 0 {
+        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
+    }
+
+    let t = unsafe { btf__type_by_id(btf, tid as _) };
+    if t.is_null() {
+        bail!("btf__type_by_id({}) returned NULL", tid);
+    }
+    let t = unsafe { &*t };
+
+    for m in btf_members(t).iter() {
+        if btf_name_str_by_offset(btf, m.name_off)? == field {
+            return Ok(true);
+        }
+    }
+
+    return Ok(false);
+}
+
+/// struct sched_ext_ops can change over time. If
+/// compat.bpf.h::SCX_OPS_DEFINE() is used to define ops and scx_ops_load!()
+/// and scx_ops_attach!() are used to load and attach it, backward
+/// compatibility is automatically maintained where reasonable.
+///
+/// - sched_ext_ops.exit_dump_len was added later. On kernels which don't
+/// support it, the value is ignored and a warning is triggered if the value
+/// is requested to be non-zero.
+#[macro_export]
+macro_rules! scx_ops_load {
+    ($skel: expr, $ops: ident, $uei: ident) => {{
+        scx_utils::paste! {
+            scx_utils::uei_set_size!($skel, $ops, $uei);
+
+            let ops = $skel.struct_ops.[<$ops _mut>]();
+            let has_field = scx_utils::compat::struct_has_field("sched_ext_ops", "exit_dump_len")?;
+            if !has_field && ops.exit_dump_len != 0 {
+                scx_utils::warn!("Kernel doesn't support setting exit dump len");
+                ops.exit_dump_len = 0;
+            }
+
+            $skel.load().context("Failed to load BPF program")
+        }
+    }};
+}
+
+/// Must be used together with scx_ops_load!(). See there.
+#[macro_export]
+macro_rules! scx_ops_attach {
+    ($skel: expr, $ops: ident) => {{
+        $skel
+            .maps_mut()
+            .$ops()
+            .attach_struct_ops()
+            .context("Failed to attach struct ops")
+    }};
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_read_enum() {
+        assert_eq!(super::read_enum("pid_type", "PIDTYPE_TGID").unwrap(), 1);
+    }
+
+    #[test]
+    fn test_struct_has_field() {
+        assert!(super::struct_has_field("task_struct", "flags").unwrap());
+        assert!(!super::struct_has_field("task_struct", "NO_SUCH_FIELD").unwrap());
+        assert!(super::struct_has_field("NO_SUCH_STRUCT", "NO_SUCH_FIELD").is_err());
+    }
+}
--- a/rust/scx_utils/src/lib.rs
+++ b/rust/scx_utils/src/lib.rs
@ -30,6 +30,9 @@
 //! Utility modules which can be useful for userspace component of sched_ext
 //! schedulers.

+pub use paste::paste;
+pub use log::warn;
+
 mod bindings;

 mod bpf_builder;
@ -38,21 +41,26 @@ pub use bpf_builder::BpfBuilder;
 mod builder;
 pub use builder::Builder;

-pub mod ravg;
+mod user_exit_info;
+pub use user_exit_info::ScxExitKind;
+pub use user_exit_info::ScxInternalConsts;
+pub use user_exit_info::UeiDumpPtr;
+pub use user_exit_info::UserExitInfo;
+pub use user_exit_info::UEI_DUMP_PTR_MUTEX;
+
+pub mod compat;

 mod libbpf_logger;
 pub use libbpf_logger::init_libbpf_logging;

-mod user_exit_info;
-pub use user_exit_info::UserExitInfo;
-pub use user_exit_info::ScxExitKind;
+pub mod ravg;

 mod topology;
-pub use topology::Topology;
-pub use topology::Cpu;
-pub use topology::Core;
 pub use topology::Cache;
+pub use topology::Core;
+pub use topology::Cpu;
 pub use topology::Node;
+pub use topology::Topology;

 mod cpumask;
 pub use cpumask::Cpumask;
--- a/rust/scx_utils/src/user_exit_info.rs
+++ b/rust/scx_utils/src/user_exit_info.rs
@ -7,6 +7,16 @@ use anyhow::bail;
 use anyhow::Result;
 use std::ffi::CStr;
 use std::os::raw::c_char;
+use std::sync::Mutex;
+
+pub struct UeiDumpPtr {
+    pub ptr: *const c_char,
+}
+unsafe impl Send for UeiDumpPtr {}
+
+pub static UEI_DUMP_PTR_MUTEX: Mutex<UeiDumpPtr> = Mutex::new(UeiDumpPtr {
+    ptr: std::ptr::null(),
+});

 pub enum ScxExitKind {
    None = bindings::scx_exit_kind_SCX_EXIT_NONE as isize,
@ -18,29 +28,63 @@ pub enum ScxExitKind {
    ErrorStall = bindings::scx_exit_kind_SCX_EXIT_ERROR_STALL as isize,
 }

+pub enum ScxInternalConsts {
+    ExitDumpDflLen = bindings::scx_internal_consts_SCX_EXIT_DUMP_DFL_LEN as isize,
+}
+
 /// Takes a reference to C struct user_exit_info and reads it into
 /// UserExitInfo. See UserExitInfo.
 #[macro_export]
 macro_rules! uei_read {
-    ($bpf_uei:expr) => {{
-        {
-            let bpf_uei = $bpf_uei;
+    ($skel: expr, $uei:ident) => {{
+        scx_utils::paste! {
+            let bpf_uei = $skel.data().$uei;
+            let bpf_dump = scx_utils::UEI_DUMP_PTR_MUTEX.lock().unwrap().ptr;
+
            scx_utils::UserExitInfo::new(
                &bpf_uei.kind as *const _,
                bpf_uei.reason.as_ptr() as *const _,
                bpf_uei.msg.as_ptr() as *const _,
-                bpf_uei.dump.as_ptr() as *const _,
+                bpf_dump,
            )
        }
    }};
 }

+/// Resize debug dump area according to ops.exit_dump_len. If this macro is
+/// not called, debug dump area is not allocated and debug dump won't be
+/// printed out.
+#[macro_export]
+macro_rules! uei_set_size {
+    ($skel: expr, $ops: ident, $uei:ident) => {{
+        scx_utils::paste! {
+            let len = match $skel.struct_ops.$ops().exit_dump_len {
+                0 => scx_utils::ScxInternalConsts::ExitDumpDflLen as u32,
+                v => v,
+            };
+            $skel.rodata_mut().[<$uei _dump_len>] = len;
+            $skel.maps_mut().[<data_ $uei _dump>]().set_value_size(len).unwrap();
+
+            let mut ptr = scx_utils::UEI_DUMP_PTR_MUTEX.lock().unwrap();
+            *ptr = scx_utils::UeiDumpPtr { ptr:
+                       $skel
+                       .maps()
+                       .[<data_ $uei _dump>]()
+                       .initial_value()
+                       .unwrap()
+                       .as_ptr() as *const _,
+            };
+        }
+    }};
+}
+
 /// Takes a reference to C struct user_exit_info and test whether the BPF
 /// scheduler has exited. See UserExitInfo.
 #[macro_export]
 macro_rules! uei_exited {
-    ($bpf_uei:expr) => {{
-        (unsafe { std::ptr::read_volatile(&$bpf_uei.kind as *const _) } != 0)
+    ($skel: expr, $uei:ident) => {{
+        let bpf_uei = $skel.data().uei;
+        (unsafe { std::ptr::read_volatile(&bpf_uei.kind as *const _) } != 0)
    }};
 }

@ -48,8 +92,8 @@ macro_rules! uei_exited {
 /// UserExitInfo::report() on it. See UserExitInfo.
 #[macro_export]
 macro_rules! uei_report {
-    ($bpf_uei:expr) => {{
-        scx_utils::uei_read!($bpf_uei).report()
+    ($skel: expr, $uei:ident) => {{
+        scx_utils::uei_read!($skel, $uei).report()
    }};
 }

@ -78,7 +122,7 @@ impl UserExitInfo {
    ) -> Self {
        let kind = unsafe { std::ptr::read_volatile(kind_ptr) };

-        let (reason, msg, dump) = (
+        let (reason, msg) = (
            Some(
                unsafe { CStr::from_ptr(reason_ptr) }
                    .to_str()
@ -93,14 +137,19 @@ impl UserExitInfo {
                    .to_string(),
            )
            .filter(|s| !s.is_empty()),
+        );
+
+        let dump = if dump_ptr.is_null() {
+            None
+        } else {
            Some(
                unsafe { CStr::from_ptr(dump_ptr) }
                    .to_str()
                    .expect("Failed to convert msg to string")
                    .to_string(),
            )
-            .filter(|s| !s.is_empty()),
-        );
+            .filter(|s| !s.is_empty())
+        };

        Self {
            kind,
@ -118,12 +167,12 @@ impl UserExitInfo {
            return Ok(());
        }

-	if let Some(dump) = &self.dump {
-	    eprintln!("\nDEBUG DUMP");
-	    eprintln!("================================================================================\n");
-	    eprintln!("{}", dump);
-	    eprintln!("================================================================================\n");
-	}
+        if let Some(dump) = &self.dump {
+            eprintln!("\nDEBUG DUMP");
+            eprintln!("================================================================================\n");
+            eprintln!("{}", dump);
+            eprintln!("================================================================================\n");
+        }

        let why = match (&self.reason, &self.msg) {
            (Some(reason), None) => format!("EXIT: {}", reason),
--- a/scheds/rust/scx_lavd/Cargo.toml
+++ b/scheds/rust/scx_lavd/Cargo.toml
@ -13,7 +13,7 @@ clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] }
 ctrlc = { version = "3.1", features = ["termination"] }
 fb_procfs = "0.7.0"
 hex = "0.4.3"
-libbpf-rs = "0.22.0"
+libbpf-rs = "0.22"
 libc = "0.2.137"
 log = "0.4.17"
 ordered-float = "3.4.0"
@ -29,3 +29,8 @@ scx_utils = { path = "../../../rust/scx_utils", version = "0.6" }

 [features]
 enable_backtrace = []
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "75042c653ee956b8c262e41ca4bcfcb0e2c461a1" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
--- a/scheds/rust/scx_layered/Cargo.toml
+++ b/scheds/rust/scx_layered/Cargo.toml
@ -27,3 +27,8 @@ scx_utils = { path = "../../../rust/scx_utils", version = "0.6" }

 [features]
 enable_backtrace = []
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "125444300e2db452131dd6453101599c1a277784" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
--- a/scheds/rust/scx_layered/src/bpf/main.bpf.c
+++ b/scheds/rust/scx_layered/src/bpf/main.bpf.c
@ -30,7 +30,7 @@ static u32 preempt_cursor;

 #include "util.bpf.c"

-struct user_exit_info uei;
+UEI_DEFINE(uei);

 static inline bool vtime_before(u64 a, u64 b)
 {
@ -894,7 +894,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(layered_init)
 	struct bpf_cpumask *cpumask;
 	int i, j, k, nr_online_cpus, ret;

-	scx_bpf_switch_all();
+	__COMPAT_scx_bpf_switch_all();

 	cpumask = bpf_cpumask_create();
 	if (!cpumask)
@ -1020,23 +1020,21 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(layered_init)

 void BPF_STRUCT_OPS(layered_exit, struct scx_exit_info *ei)
 {
-	uei_record(&uei, ei);
+	UEI_RECORD(uei, ei);
 }

-SEC(".struct_ops.link")
-struct sched_ext_ops layered = {
-	.select_cpu		= (void *)layered_select_cpu,
-	.enqueue		= (void *)layered_enqueue,
-	.dispatch		= (void *)layered_dispatch,
-	.runnable		= (void *)layered_runnable,
-	.running		= (void *)layered_running,
-	.stopping		= (void *)layered_stopping,
-	.quiescent		= (void *)layered_quiescent,
-	.set_weight		= (void *)layered_set_weight,
-	.set_cpumask		= (void *)layered_set_cpumask,
-	.init_task		= (void *)layered_init_task,
-	.exit_task		= (void *)layered_exit_task,
-	.init			= (void *)layered_init,
-	.exit			= (void *)layered_exit,
-	.name			= "layered",
-};
+SCX_OPS_DEFINE(layered,
+	       .select_cpu		= (void *)layered_select_cpu,
+	       .enqueue			= (void *)layered_enqueue,
+	       .dispatch		= (void *)layered_dispatch,
+	       .runnable		= (void *)layered_runnable,
+	       .running			= (void *)layered_running,
+	       .stopping		= (void *)layered_stopping,
+	       .quiescent		= (void *)layered_quiescent,
+	       .set_weight		= (void *)layered_set_weight,
+	       .set_cpumask		= (void *)layered_set_cpumask,
+	       .init_task		= (void *)layered_init_task,
+	       .exit_task		= (void *)layered_exit_task,
+	       .init			= (void *)layered_init,
+	       .exit			= (void *)layered_exit,
+	       .name			= "layered");
--- a/scheds/rust/scx_layered/src/main.rs
+++ b/scheds/rust/scx_layered/src/main.rs
@ -29,7 +29,6 @@ use anyhow::Result;
 use bitvec::prelude::*;
 use clap::Parser;
 use libbpf_rs::skel::OpenSkel as _;
-use libbpf_rs::skel::Skel as _;
 use libbpf_rs::skel::SkelBuilder as _;
 use log::debug;
 use log::info;
@ -41,6 +40,8 @@ use prometheus_client::metrics::gauge::Gauge;
 use prometheus_client::registry::Registry;
 use scx_utils::init_libbpf_logging;
 use scx_utils::ravg::ravg_read;
+use scx_utils::scx_ops_attach;
+use scx_utils::scx_ops_load;
 use scx_utils::uei_exited;
 use scx_utils::uei_report;
 use serde::Deserialize;
@ -271,6 +272,10 @@ struct Opts {
    #[clap(short = 'n', long)]
    no_load_frac_limit: bool,

+    /// Exit debug dump buffer length. 0 indicates default.
+    #[clap(long, default_value = "0")]
+    exit_dump_len: u32,
+
    /// Enable verbose output including libbpf details. Specify multiple
    /// times to increase verbosity.
    #[clap(short = 'v', long, action = clap::ArgAction::Count)]
@ -303,28 +308,28 @@ enum LayerMatch {
 enum LayerKind {
    Confined {
        util_range: (f64, f64),
-	#[serde(default)]
+        #[serde(default)]
        cpus_range: Option<(usize, usize)>,
-	#[serde(default)]
+        #[serde(default)]
        min_exec_us: u64,
    },
    Grouped {
        util_range: (f64, f64),
-	#[serde(default)]
+        #[serde(default)]
        cpus_range: Option<(usize, usize)>,
-	#[serde(default)]
+        #[serde(default)]
        min_exec_us: u64,
-	#[serde(default)]
+        #[serde(default)]
        preempt: bool,
-	#[serde(default)]
+        #[serde(default)]
        exclusive: bool,
    },
    Open {
-	#[serde(default)]
+        #[serde(default)]
        min_exec_us: u64,
-	#[serde(default)]
+        #[serde(default)]
        preempt: bool,
-	#[serde(default)]
+        #[serde(default)]
        exclusive: bool,
    },
 }
@ -1282,10 +1287,10 @@ impl<'a> Scheduler<'a> {
                    exclusive,
                    ..
                } => {
-                    layer.open = true;
+                    layer.open.write(true);
                    layer.min_exec_ns = min_exec_us * 1000;
-                    layer.preempt = *preempt;
-                    layer.exclusive = *exclusive;
+                    layer.preempt.write(*preempt);
+                    layer.exclusive.write(*exclusive);
                }
            }
        }
@ -1304,6 +1309,8 @@ impl<'a> Scheduler<'a> {
        let mut skel = skel_builder.open().context("Failed to open BPF program")?;

        // Initialize skel according to @opts.
+        skel.struct_ops.layered_mut().exit_dump_len = opts.exit_dump_len;
+
        skel.rodata_mut().debug = opts.verbose as u32;
        skel.rodata_mut().slice_ns = opts.slice_us * 1000;
        skel.rodata_mut().nr_possible_cpus = *NR_POSSIBLE_CPUS as u32;
@ -1316,7 +1323,8 @@ impl<'a> Scheduler<'a> {
        }
        Self::init_layers(&mut skel, &layer_specs)?;

-        let mut skel = skel.load().context("Failed to load BPF program")?;
+        let mut skel = scx_ops_load!(skel, layered, uei)?;
+
        let mut layers = vec![];
        for spec in layer_specs.iter() {
            layers.push(Layer::new(&mut cpu_pool, &spec.name, spec.kind.clone())?);
@ -1357,24 +1365,13 @@ impl<'a> Scheduler<'a> {
        // huge problem in the interim until we figure it out.

        // Attach.
-        sched
-            .skel
-            .attach()
-            .context("Failed to attach BPF program")?;
-        sched.struct_ops = Some(
-            sched
-                .skel
-                .maps_mut()
-                .layered()
-                .attach_struct_ops()
-                .context("Failed to attach layered struct ops")?,
-        );
+        sched.struct_ops = Some(scx_ops_attach!(sched.skel, layered)?);
        info!("Layered Scheduler Attached");

        Ok(sched)
    }

-    fn update_bpf_layer_cpumask(layer: &Layer, bpf_layer: &mut bpf_bss_types::layer) {
+    fn update_bpf_layer_cpumask(layer: &Layer, bpf_layer: &mut bpf_types::layer) {
        for bit in 0..layer.cpus.len() {
            if layer.cpus[bit] {
                bpf_layer.cpus[bit / 8] |= 1 << (bit % 8);
@ -1709,7 +1706,7 @@ impl<'a> Scheduler<'a> {
        let mut next_sched_at = now + self.sched_intv;
        let mut next_monitor_at = now + self.monitor_intv;

-        while !shutdown.load(Ordering::Relaxed) && !uei_exited!(&self.skel.bss().uei) {
+        while !shutdown.load(Ordering::Relaxed) && !uei_exited!(&self.skel, uei) {
            let now = Instant::now();

            if now >= next_sched_at {
@ -1734,7 +1731,7 @@ impl<'a> Scheduler<'a> {
        }

        self.struct_ops.take();
-        uei_report!(&self.skel.bss().uei)
+        uei_report!(&self.skel, uei)
    }
 }

--- a/scheds/rust/scx_rlfifo/Cargo.toml
+++ b/scheds/rust/scx_rlfifo/Cargo.toml
@ -9,7 +9,7 @@ license = "GPL-2.0-only"
 [dependencies]
 anyhow = "1.0.65"
 ctrlc = { version = "3.1", features = ["termination"] }
-libbpf-rs = "0.22.0"
+libbpf-rs = "0.22"
 libc = "0.2.137"
 scx_utils = { path = "../../../rust/scx_utils", version = "0.6" }
 scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "0.1" }
@ -20,3 +20,8 @@ scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "0.1"

 [features]
 enable_backtrace = []
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "75042c653ee956b8c262e41ca4bcfcb0e2c461a1" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
--- a/scheds/rust/scx_rustland/Cargo.toml
+++ b/scheds/rust/scx_rustland/Cargo.toml
@ -11,7 +11,7 @@ anyhow = "1.0.65"
 clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] }
 ctrlc = { version = "3.1", features = ["termination"] }
 fb_procfs = "0.7.0"
-libbpf-rs = "0.22.0"
+libbpf-rs = "0.22"
 libc = "0.2.137"
 log = "0.4.17"
 ordered-float = "3.4.0"
@ -25,3 +25,8 @@ scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "0.1"

 [features]
 enable_backtrace = []
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "75042c653ee956b8c262e41ca4bcfcb0e2c461a1" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }
--- a/scheds/rust/scx_rusty/Cargo.toml
+++ b/scheds/rust/scx_rusty/Cargo.toml
@ -11,7 +11,7 @@ anyhow = "1.0.65"
 clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] }
 ctrlc = { version = "3.1", features = ["termination"] }
 fb_procfs = "0.7.0"
-libbpf-rs = "0.22.0"
+libbpf-rs = "0.22"
 libc = "0.2.137"
 log = "0.4.17"
 ordered-float = "3.4.0"
@ -25,3 +25,8 @@ scx_utils = { path = "../../../rust/scx_utils", version = "0.6" }

 [features]
 enable_backtrace = []
+
+[patch.crates-io]
+libbpf-sys = { git = "https://github.com/libbpf/libbpf-sys.git", rev = "75042c653ee956b8c262e41ca4bcfcb0e2c461a1" }
+libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "560641a5aff2c613dff6ae02147b0000558e4945" }
+libbpf-cargo = { git = "https://github.com/libbpf/libbpf-rs.git", rev = "7745f2e43c2bde59920b8b7400d62f66b906f05b" }