From fb442630d0ca02ff864527747670a7553dc9c776 Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Mon, 21 Oct 2024 13:20:44 +0100 Subject: [PATCH] tests: add integration testing framework --- Cargo.lock | 227 +++++++++++++++ Cargo.toml | 2 +- .../scx_integration_test_framework/Cargo.toml | 16 ++ .../src/builder.rs | 112 ++++++++ .../scx_integration_test_framework/src/lib.rs | 258 ++++++++++++++++++ .../src/main.rs | 86 ++++++ .../src/runner_common.rs | 41 +++ .../src/target_common.rs | 128 +++++++++ tests/scx_layered_tests/Cargo.toml | 12 + tests/scx_layered_tests/build.rs | 14 + tests/scx_layered_tests/src/lib.rs | 0 tests/scx_layered_tests/src/main.rs | 1 + tests/scx_layered_tests/tests/integration.rs | 5 + .../tests/layer_growth_random.toml | 26 ++ 14 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 tests/scx_integration_test_framework/Cargo.toml create mode 100644 tests/scx_integration_test_framework/src/builder.rs create mode 100644 tests/scx_integration_test_framework/src/lib.rs create mode 100644 tests/scx_integration_test_framework/src/main.rs create mode 100644 tests/scx_integration_test_framework/src/runner_common.rs create mode 100644 tests/scx_integration_test_framework/src/target_common.rs create mode 100644 tests/scx_layered_tests/Cargo.toml create mode 100644 tests/scx_layered_tests/build.rs create mode 100644 tests/scx_layered_tests/src/lib.rs create mode 100644 tests/scx_layered_tests/src/main.rs create mode 100644 tests/scx_layered_tests/tests/integration.rs create mode 100644 tests/scx_layered_tests/tests/layer_growth_random.toml diff --git a/Cargo.lock b/Cargo.lock index daa54bf..debed1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,6 +255,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bindgen" version = "0.69.4" @@ -524,6 +530,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const_format" version = "0.2.31" @@ -704,6 +723,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "endi" version = "1.1.0" @@ -737,6 +762,19 @@ dependencies = [ "syn", ] +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -954,6 +992,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -993,12 +1037,32 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1494,6 +1558,72 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "qapi" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6412bdd014ebee03ddbbe79ac03a0b622cce4d80ba45254f6357c847f06fa38" +dependencies = [ + "log", + "qapi-qga", + "qapi-qmp", + "qapi-spec", + "serde", + "serde_json", +] + +[[package]] +name = "qapi-codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb959fed63a69baa2e3ae57224d885e686bc3f56c9bb3b03406969980ea57a44" +dependencies = [ + "qapi-parser", +] + +[[package]] +name = "qapi-parser" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b37f643cfdf67a409a9323334138a11636a5db5d56cedcc780d7a82a7fb7659" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "qapi-qga" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c54b6cbbc1b102dfebad74155799d7d96eeb3654eb311f330d6ff8c3177933e" +dependencies = [ + "qapi-codegen", + "qapi-spec", + "serde", +] + +[[package]] +name = "qapi-qmp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b944db7e544d2fa97595e9a000a6ba5c62c426fa185e7e00aabe4b5640b538" +dependencies = [ + "qapi-codegen", + "qapi-spec", + "serde", +] + +[[package]] +name = "qapi-spec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e6bdbbe5d13015b21a49a778a29ae3cee9c450c3154e1648aed670d57fe5ba" +dependencies = [ + "base64", + "serde", + "serde_json", +] + [[package]] name = "quanta" version = "0.12.3" @@ -1683,6 +1813,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scx_bpfland" version = "1.0.5" @@ -1700,6 +1836,21 @@ dependencies = [ "simplelog", ] +[[package]] +name = "scx_integration_test_framework" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "libc", + "scx_layered", + "serde", + "serde_json", + "tempfile", + "toml 0.8.19", + "vmtest", +] + [[package]] name = "scx_lavd" version = "1.0.5" @@ -1750,6 +1901,14 @@ dependencies = [ "simplelog", ] +[[package]] +name = "scx_layered_tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "scx_integration_test_framework", +] + [[package]] name = "scx_loader" version = "1.0.5" @@ -1967,6 +2126,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2234,6 +2402,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.40.0" @@ -2263,11 +2441,35 @@ dependencies = [ "syn", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -2276,6 +2478,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -2393,6 +2597,29 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vmtest" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838ad52ea690d452c6c815d5f8cffb31d37eb61e1c82818511ede822b4bd83d" +dependencies = [ + "anyhow", + "clap", + "console", + "env_logger", + "itertools 0.10.5", + "log", + "qapi", + "rand", + "regex", + "scopeguard", + "serde", + "serde_derive", + "tempfile", + "tinytemplate", + "toml 0.5.11", +] + [[package]] name = "vsprintf" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 577a326..adc3646 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,5 +10,5 @@ members = ["rust/scx_stats", "scheds/rust/scx_rlfifo", "scheds/rust/scx_rusty", "scheds/rust/scx_layered", - "scheds/rust/scx_mitosis"] + "scheds/rust/scx_mitosis", "tests/scx_integration_test_framework", "tests/scx_layered_tests"] resolver = "2" diff --git a/tests/scx_integration_test_framework/Cargo.toml b/tests/scx_integration_test_framework/Cargo.toml new file mode 100644 index 0000000..cf5d3dc --- /dev/null +++ b/tests/scx_integration_test_framework/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "scx_integration_test_framework" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.65" +clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } +serde = { version = "1.0", features = ["derive"] } +toml = "0.8" +libc = "0.2.137" +serde_json = "1.0" +tempfile = "3" +vmtest = "0.14.0" + +scx_layered = { path = "../../scheds/rust/scx_layered" } diff --git a/tests/scx_integration_test_framework/src/builder.rs b/tests/scx_integration_test_framework/src/builder.rs new file mode 100644 index 0000000..c5db04c --- /dev/null +++ b/tests/scx_integration_test_framework/src/builder.rs @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +use anyhow::bail; + +use std::path::{Path, PathBuf}; +use std::env; +use std::fs; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; + +pub struct Builder { + out_dir: PathBuf, + target_filename: Option, + runner_filename: Option, + + + test_paths: Vec, +} + +fn read_test_config_from_toml(path: &Path) -> anyhow::Result { + let content = std::fs::read_to_string(&path)?; + Ok(toml::from_str(&content)?) +} + +impl Builder { + +pub fn new() -> anyhow::Result { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + Ok(Builder{ + out_dir, + target_filename: None, + runner_filename: None, + + test_paths: vec![], + }) +} + +pub fn register_test(&mut self, path: PathBuf) ->&mut Self { + println!("cargo::rerun-if-changed={}", path.display()); + self.test_paths.push(path); + self +} + +pub fn register_test_dir(&mut self, test_dir: &Path) -> anyhow::Result<&mut Self> { + println!("cargo::rerun-if-changed={}", test_dir.display()); + + let mut added = false; + for entry in fs::read_dir(test_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_file() && path.extension().is_some_and(|e| e == "toml") { + self.test_paths.push(path); + added = true; + } + } + + if !added { + println!("cargo::warning=Test directory `{}` provided doesn't contain any .toml files", test_dir.display()); + } + + Ok(self) +} + +pub fn enable_target(&mut self, path: PathBuf) -> anyhow::Result<&mut Self> { + if !path.is_relative() { + bail!("path supplied must be relative: `{}`", path.display()); + } + self.target_filename = Some(path); + Ok(self) +} + +pub fn enable_runner(&mut self, path: PathBuf) -> anyhow::Result<&mut Self> { + if !path.is_relative() { + bail!("path supplied must be relative: `{}`", path.display()); + } + self.runner_filename = Some(path); + Ok(self) +} + + +pub fn build(&self) -> anyhow::Result<()> { + let configs = self.test_paths.iter() + .map(|p| { + let cfg = read_test_config_from_toml(p)?; + let suite_name = p.file_stem().unwrap().to_string_lossy().into_owned(); + Ok((suite_name, cfg)) + }).collect::>>()?; + + if let Some(target_path) = &self.target_filename { + let src = crate::generate_target(&configs)?; + + let target_path = Path::join(&self.out_dir, target_path); + fs::write(target_path, src)?; + }; + if let Some(runner_path) = &self.runner_filename { + let runner_path = Path::join(&self.out_dir, runner_path); + + let mut f = OpenOptions::new().create(true).write(true).truncate(true).open(runner_path)?; + for (suite_name, cfg) in &configs { + let src = crate::generate_runner(suite_name, cfg)?; + f.write_all(src.as_bytes())?; + } + + }; + + Ok(()) +} + +} diff --git a/tests/scx_integration_test_framework/src/lib.rs b/tests/scx_integration_test_framework/src/lib.rs new file mode 100644 index 0000000..6eda4cd --- /dev/null +++ b/tests/scx_integration_test_framework/src/lib.rs @@ -0,0 +1,258 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +pub mod runner_common; +pub mod target_common; + +pub use builder::Builder; + +mod builder; + +use serde::Deserialize; + +use std::collections::HashMap; + +const fn default_u8() -> u8 { + V +} +const fn default_u32() -> u32 { + V +} + +#[derive(Deserialize)] +pub struct TestConfig { + pub topology: Topology, + pub workload: Workload, + pub scheduler: Scheduler, + pub cases: HashMap, +} + +#[derive(Deserialize)] +pub struct Topology { + #[serde(default = "default_u8::<1>")] + pub sockets: u8, + #[serde(default = "default_u8::<1>")] + pub llcs_per_socket: u8, + #[serde(default = "default_u8::<2>")] + pub cores_per_llc: u8, + #[serde(default = "default_u8::<2>")] + pub threads_per_core: u8, +} + +#[derive(Deserialize)] +#[serde(tag = "type")] +pub enum Workload { + #[serde(rename = "stress-ng")] + StressNg { args: Vec }, +} + +#[derive(Deserialize)] +#[serde(tag = "type")] +#[serde(rename_all = "lowercase")] +pub enum Scheduler { + Layered { + #[serde(default)] + args: Vec, + + config: scx_layered::LayerConfig, + }, +} + +#[derive(Deserialize)] +pub struct Case { + /// Time to delay the test for after starting the scheduler and workload. + #[serde(default = "default_u32::<5>")] + pub delay_s: u32, + + #[serde(flatten)] + pub test: CaseTest, +} + +#[derive(Deserialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum CaseTest { + Bpftrace { script: String, expect_json: String }, +} + +impl Topology { + pub(crate) fn num_cpus(&self) -> u8 { + self.sockets * self.llcs_per_socket * self.cores_per_llc + } +} + +fn codegen_vec_of_strings(strings: &[String]) -> String { + let mut codegen = String::new(); + + codegen.push_str("vec![ "); + for s in strings { + codegen.push('"'); + for c in s.chars() { + if c == '"' { + codegen.push('\\'); + } + codegen.push(c); + } + codegen.push_str("\", "); + } + codegen.push(']'); + + codegen +} + +fn generate_workload(workload: &Workload) -> String { + match workload { + Workload::StressNg { args } => { + let mut codegen = String::new(); + + codegen.push_str(" let args: Vec<&str> = "); + codegen.push_str(&codegen_vec_of_strings(args)); + codegen.push_str(";\n"); + + // TODO: replace stress-ng with something that isn't constant, maybe an env var? it + // depends how we do the mounting in the VM really. + codegen.push_str(" let mut c = std::process::Command::new(\"stress-ng\");\n"); + codegen.push_str(" c.args(args);\n"); + codegen.push_str(" Ok(target_common::WorkloadHandle::StressNg(target_common::StressNgWorkload::new(c).expect(\"failed to start stress-ng\")))\n"); + + codegen + } + } +} + +fn generate_scheduler(scheduler: &Scheduler) -> anyhow::Result { + match scheduler { + Scheduler::Layered { args, config } => { + // TODO: change me + const LAYERED: &str = "/data/users/jakehillion/scx/target/release/scx_layered"; + + let mut codegen = String::new(); + codegen.push_str(" use std::io::Write;\n"); + codegen.push_str(" use std::os::fd::AsRawFd;\n"); + + codegen.push_str(" let args: Vec<&str> = "); + codegen.push_str(&codegen_vec_of_strings(args)); + codegen.push_str(";\n"); + + codegen.push_str(" let cfg = r#\""); + codegen.push_str(&serde_json::to_string(config)?); + codegen.push_str("\"#;\n"); + + codegen.push_str(" let mut cfg_file = target_common::tempfile::tempfile()?;\n"); + codegen.push_str(" cfg_file.write_all(cfg.as_bytes())?;\n"); + + codegen.push_str(" let child = std::process::Command::new(\""); + codegen.push_str(LAYERED); + codegen.push_str("\").args(args).arg(format!(\"f:/proc/{}/fd/{}\", unsafe { target_common::libc::getpid() }, cfg_file.as_raw_fd())).spawn()?;\n"); + + // TODO: this is racy as anything - is there a better way to check the scheduler has + // started? the scheduler has to start before the tempfile is dropped. + codegen.push_str(" std::thread::sleep(std::time::Duration::from_millis(100));\n"); + + codegen.push_str(" Ok(target_common::SchedulerHandle::new(child))\n"); + + Ok(codegen) + }, + } +} + +/// Generate target, which will go in their crate/main.rs. +pub fn generate_target(cfgs: &[(String, TestConfig)]) -> anyhow::Result { + let mut codegen = String::new(); + + codegen.push_str("use scx_integration_test_framework::target_common;\n\n"); + + // Generate setup functions + for (suite_name, cfg) in cfgs { + codegen.push_str(&format!("fn run_workload_{}() -> anyhow::Result {{\n", suite_name)); + codegen.push_str(&generate_workload(&cfg.workload)); + codegen.push_str("}\n"); + + codegen.push_str(&format!("fn run_scheduler_{}() -> anyhow::Result {{\n", suite_name)); + codegen.push_str(&generate_scheduler(&cfg.scheduler)?); + codegen.push_str("}\n"); + } + + // Generate per-case targets + for (suite_name, cfg) in cfgs { + for (case_name, case) in &cfg.cases { + let case_full_name = format!("{}_{}", suite_name, case_name); + codegen.push_str(&format!("fn {}_target() {{\n", case_full_name)); + + codegen.push_str(&format!(" let mut workload = run_workload_{}().expect(\"failed to start workload\");\n", suite_name)); + codegen.push_str(&format!(" let mut scheduler = run_scheduler_{}().expect(\"failed to start workload\");\n", suite_name)); + + let delay_ms = std::cmp::max(case.delay_s * 1000, 100); + codegen.push_str(&format!(" println!(\"workload & scheduler started, sleeping for {}ms while they warm up\");\n", delay_ms)); + codegen.push_str(&format!(" std::thread::sleep(std::time::Duration::from_millis({}));\n", delay_ms)); + + codegen.push_str(" assert!(workload.is_alive().unwrap(), \"workload stopped prematurely\");\n"); + codegen.push_str(" assert!(scheduler.is_alive().unwrap(), \"workload stopped prematurely\");\n"); + + // TODO: run test + + codegen.push_str(" workload.cleanup().expect(\"workload failed to clean up\");\n"); + codegen.push_str(" scheduler.cleanup().expect(\"scheduler failed to clean up\");\n"); + codegen.push_str("}\n"); + } + } + + codegen.push('\n'); + + // Generate main function distributing by argument + codegen.push_str("fn main() -> anyhow::Result<()> {\n"); + codegen.push_str(" let arg = std::env::args().nth(1).expect(\"case argument required\");\n"); + codegen.push_str(" match arg.as_str() {\n"); + for (suite_name, cfg) in cfgs { + for (case_name, _) in &cfg.cases { + let case_full_name = format!("{}_{}", suite_name, case_name); + codegen.push_str(&format!(" \"{0}\" => {0}_target(),\n", case_full_name)); + } + } + codegen.push_str(" &_ => anyhow::bail!(\"invalid case name: {}\", arg.as_str()),\n"); + codegen.push_str(" }\n Ok(())\n}\n"); + + Ok(codegen) +} + +/// Generate runner, which will go in their crate/test/integration.rs with a module per suite (toml +/// file). +pub fn generate_runner(suite_name: &str, cfg: &TestConfig) -> anyhow::Result { + let mut codegen = String::new(); + + codegen.push_str(&format!("mod {} {{", suite_name)); + codegen.push_str("use scx_integration_test_framework::runner_common;\n\n"); + + // Constants + codegen.push_str("const TARGET_BINARY: &str = env!(concat!(\"CARGO_BIN_EXE_\", env!(\"CARGO_PKG_NAME\")));\n"); + + // Generate runner function for full suite + codegen.push_str("fn run() -> anyhow::Result<()> {\n"); + // TODO: setup VM and run the `target` in it, returning the result which probably needs to be + // an enum + codegen.push_str(" todo!(\"run\")\n"); + codegen.push_str("}\n\n"); + + // Generate test cases + for (case_name, case) in &cfg.cases { + codegen.push_str("#[test]\n"); + + codegen.push_str(&format!("fn {}() {{\n", case_name)); + + // codegen.push_str(" let _ = setup();\n"); + + + let case_full_name = format!("{}_{}", suite_name, case_name); + // codegen.push_str(&format!(" let target_status = std::process::Command::new(TARGET_BINARY).arg(\"{}\").spawn().expect(\"failed to spawn test target\").wait().expect(\"failed to wait for test target\");\n", case_full_name)); + codegen.push_str(&format!(" let target_status = runner_common::run_target_in_vm();\n", case_full_name)); + codegen.push_str(" assert!(target_status.success(), \"target failed with exit code {:?}\", target_status.code());\n"); + + // codegen.push_str(" todo!(\"parse and assert the results from the target\");\n"); + + codegen.push_str("}\n\n"); + } + + codegen.push_str("}\n"); + Ok(codegen) +} diff --git a/tests/scx_integration_test_framework/src/main.rs b/tests/scx_integration_test_framework/src/main.rs new file mode 100644 index 0000000..02fb7ed --- /dev/null +++ b/tests/scx_integration_test_framework/src/main.rs @@ -0,0 +1,86 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +use anyhow::bail; +use clap::{Parser, Subcommand}; + +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use scx_integration_test_framework::TestConfig; + +#[derive(Parser)] +#[command(verbatim_doc_comment)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + #[command(subcommand)] + Generate(Generate), +} + +#[derive(Subcommand)] +enum Generate { + Target { + paths: Vec, + }, + Runner { + /// TOML spec to build test cases from + path: PathBuf, + }, +} + +fn read_test_config_from_toml(path: &Path) -> anyhow::Result { + if !path.is_file() { + bail!("path provided must be a filename!"); + } + + let content = std::fs::read_to_string(&path)?; + Ok(toml::from_str(&content)?) +} + +impl Generate { + fn run(&self) -> anyhow::Result<()> { + match self { + Generate::Target { paths } => { + let cfgs = paths + .iter() + .map(|p| { + let cfg = read_test_config_from_toml(p)?; + let suite_name = p.file_stem().unwrap().to_string_lossy().into_owned(); + Ok((suite_name, cfg)) + }) + .collect::>>()?; + + let target = scx_integration_test_framework::generate_target(&cfgs)?; + println!("{}", target); + + Ok(()) + } + + Generate::Runner { path } => { + let cfg = read_test_config_from_toml(&path)?; + let runner = scx_integration_test_framework::generate_runner( + &*path.file_stem().unwrap().to_string_lossy(), + &cfg, + )?; + println!("{}", runner); + + Ok(()) + } + } + } +} + +fn main() -> anyhow::Result<()> { + let opts = Cli::parse(); + + match opts.command { + Commands::Generate(sub) => sub.run(), + } +} diff --git a/tests/scx_integration_test_framework/src/runner_common.rs b/tests/scx_integration_test_framework/src/runner_common.rs new file mode 100644 index 0000000..f4cb2e4 --- /dev/null +++ b/tests/scx_integration_test_framework/src/runner_common.rs @@ -0,0 +1,41 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +use vmtest::vmtest::Vmtest; + +use crate::Topology; + +use std::sync::mpsc::channel; +use std::path::PathBuf; +use std::collections::HashMap; + +pub fn run_target_in_vm(topo: &Topology, target_binary: std::path::PathBuf, test_case: &str) -> anyhow::Result { + let cfg = vmtest::config::Config { + target: vec![vmtest::config::Target { + name: "TBD".into(), + image: None, + uefi: false, + kernel: Some("/data/users/jakehillion/linux/arch/x86/boot/bzImage".into()), + kernel_args: None, + rootfs: "TBD".into(), + arch: "x86_64".into(), + command: "TBD".into(), + vm: vmtest::config::VMConfig { + num_cpus: topo.num_cpus(), + memory: "4G".into(), + mounts: HashMap::new(), + bios: None, + extra_args: vec![ + format!("-smp sockets={},modules={},cores={},threads={}", topo.sockets, topo.llcs_per_socket, topo.cores_per_llc, topo.threads_per_core), + ], + }, + }], + }; + let vm = Vmtest::new("/", cfg)?; + + let (tx, rx) = channel(); + vm.run_one(0, tx); + + todo!("run_test_in_vm"); +} diff --git a/tests/scx_integration_test_framework/src/target_common.rs b/tests/scx_integration_test_framework/src/target_common.rs new file mode 100644 index 0000000..245810a --- /dev/null +++ b/tests/scx_integration_test_framework/src/target_common.rs @@ -0,0 +1,128 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +pub use libc; +pub use tempfile; + +use anyhow::Context; + +use std::os::unix::process::ExitStatusExt; + +pub enum WorkloadHandle { + StressNg(StressNgWorkload), +} + +pub struct SchedulerHandle { + child: std::process::Child, +} + +pub struct StressNgWorkload { + child: std::process::Child, +} + +fn exit_status_to_error(ec: std::process::ExitStatus) -> anyhow::Result<()> { + if ec.signal() == Some(15) /* SIGTERM */ || ec.success() { + Ok(()) + } else { + Err(anyhow::anyhow!("bad exit code: {}", ec)) + } +} + +fn wait_with_timeout(child: &mut std::process::Child, frequency: std::time::Duration, attempts: u16) -> anyhow::Result<()> { + for _ in 0..attempts { + if let Some(ec) = child.try_wait().unwrap() { + return exit_status_to_error(ec); + } + std::thread::sleep(frequency); + } + + Err(anyhow::anyhow!("failed to kill stress-ng process")) +} + +fn send_sigterm(child: &std::process::Child) -> std::io::Result<()> { + if unsafe { libc::kill(child.id().try_into().unwrap(), libc::SIGTERM) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +impl WorkloadHandle { + pub fn cleanup(self) -> anyhow::Result<()> { + match self { + WorkloadHandle::StressNg(w) => w.cleanup(), + } + } + + pub fn is_alive(&mut self) -> anyhow::Result { + match self { + WorkloadHandle::StressNg(w) => w.is_alive(), + } + } +} + +impl StressNgWorkload { + +pub fn new(mut cmd: std::process::Command) -> anyhow::Result { + let child = cmd.spawn()?; + Ok(StressNgWorkload{ child }) +} + +pub fn cleanup(mut self) -> anyhow::Result<()> { + self.cleanup_impl() +} + +pub fn is_alive(&mut self) -> anyhow::Result { + Ok(self.child.try_wait()?.is_none()) +} + +fn cleanup_impl(&mut self) -> anyhow::Result<()> { + if let Some(ec) = self.child.try_wait()? { + return exit_status_to_error(ec).with_context(|| "workload terminated early"); + } + + send_sigterm(&self.child)?; + wait_with_timeout(&mut self.child, std::time::Duration::from_millis(100), 10).with_context(|| "workload failed to exit cleanly after SIGTERM sent") +} + +} + +impl Drop for StressNgWorkload { +fn drop(&mut self) { + // prefer calling cleanup as panicking in Drop is not ideal + self.cleanup_impl().unwrap(); +} +} + +impl SchedulerHandle { + +pub fn new(child: std::process::Child) -> Self { + Self { child } +} + +pub fn cleanup(mut self) -> anyhow::Result<()> { + self.cleanup_impl() +} + +pub fn cleanup_impl(&mut self) -> anyhow::Result<()> { + if let Some(ec) = self.child.try_wait()? { + return exit_status_to_error(ec).with_context(|| "scheduler terminated early"); + } + + send_sigterm(&self.child)?; + wait_with_timeout(&mut self.child, std::time::Duration::from_millis(100), 10).with_context(|| "scheduler failed to exit cleanly after SIGTERM sent") +} + +pub fn is_alive(&mut self) -> anyhow::Result { + Ok(self.child.try_wait()?.is_none()) +} + +} + +impl Drop for SchedulerHandle { +fn drop(&mut self) { + // prefer calling cleanup as panicking in Drop is not ideal + self.cleanup_impl().unwrap(); +} +} diff --git a/tests/scx_layered_tests/Cargo.toml b/tests/scx_layered_tests/Cargo.toml new file mode 100644 index 0000000..82bf48a --- /dev/null +++ b/tests/scx_layered_tests/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "scx_layered_tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.65" +scx_integration_test_framework = { path = "../scx_integration_test_framework" } + +[build-dependencies] +anyhow = "1.0.65" +scx_integration_test_framework = { path = "../scx_integration_test_framework" } diff --git a/tests/scx_layered_tests/build.rs b/tests/scx_layered_tests/build.rs new file mode 100644 index 0000000..0d0be62 --- /dev/null +++ b/tests/scx_layered_tests/build.rs @@ -0,0 +1,14 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. + +fn main() -> anyhow::Result<()> { + scx_integration_test_framework::Builder::new()? + .register_test_dir(&std::path::PathBuf::from("tests"))? + .enable_runner("generated_tests.rs".into())? + .enable_target("target.rs".into())? + .build()?; + + Ok(()) +} diff --git a/tests/scx_layered_tests/src/lib.rs b/tests/scx_layered_tests/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/tests/scx_layered_tests/src/main.rs b/tests/scx_layered_tests/src/main.rs new file mode 100644 index 0000000..d540f4b --- /dev/null +++ b/tests/scx_layered_tests/src/main.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/target.rs")); diff --git a/tests/scx_layered_tests/tests/integration.rs b/tests/scx_layered_tests/tests/integration.rs new file mode 100644 index 0000000..d05f4b2 --- /dev/null +++ b/tests/scx_layered_tests/tests/integration.rs @@ -0,0 +1,5 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +include!(concat!(env!("OUT_DIR"), "/generated_tests.rs")); diff --git a/tests/scx_layered_tests/tests/layer_growth_random.toml b/tests/scx_layered_tests/tests/layer_growth_random.toml new file mode 100644 index 0000000..20babad --- /dev/null +++ b/tests/scx_layered_tests/tests/layer_growth_random.toml @@ -0,0 +1,26 @@ +[topology] + +[workload] +type = "stress-ng" +args = ["--cpu", "40", "--cpu-load", "65"] + +[scheduler] +type = "layered" + +[[scheduler.config]] +name = "all" +comment = "all tasks" +matches = [[]] +kind = { Open = { min_exec_us = 0, slice_us = 2000, preempt = false, exclusive = false, growth_algo = "Random" } } + +[cases] + [cases.random_growth] + delay_s = 1 + type = "bpftrace" + script = ''' + BEGIN { exit(); } + ''' + expect_json = ''' + {} + ''' +