From bd68e230b96162fc1c7c01a2d74d8198401e79c9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 24 Aug 2024 21:28:51 -1000 Subject: [PATCH] scx_bpfland: Convert to scx_stats Use scx_stats instead of prometheus for stats reporting. This has a few advantages: - Stats metadata can be defined more succinctly. - Natural support for nesting statistics which will be useful in making scheduler components composable. - Support for multiple programmable readers where each reader can use their own reading interval. - Built-in stats help message generation. - Openmetrics integration is still available through scx_stats/scripts/scxstats_to_openmetrics.py. --- scheds/rust/scx_bpfland/Cargo.lock | 706 +-------------------------- scheds/rust/scx_bpfland/Cargo.toml | 6 +- scheds/rust/scx_bpfland/src/main.rs | 197 ++++---- scheds/rust/scx_bpfland/src/stats.rs | 74 +++ 4 files changed, 182 insertions(+), 801 deletions(-) create mode 100644 scheds/rust/scx_bpfland/src/stats.rs diff --git a/scheds/rust/scx_bpfland/Cargo.lock b/scheds/rust/scx_bpfland/Cargo.lock index 95ed213..d03666f 100644 --- a/scheds/rust/scx_bpfland/Cargo.lock +++ b/scheds/rust/scx_bpfland/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "ahash" version = "0.8.11" @@ -93,66 +78,12 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "aws-lc-rs" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" -dependencies = [ - "aws-lc-sys", - "mirai-annotations", - "paste", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", - "libc", - "paste", -] - -[[package]] -name = "backtrace" -version = "0.3.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "bindgen" version = "0.69.4" @@ -201,12 +132,6 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "bytes" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" - [[package]] name = "camino" version = "1.1.8" @@ -259,8 +184,6 @@ version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68064e60dbf1f17005c2fde4d07c16d8baa506fd7ffed8ccab702d93617975c7" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -339,15 +262,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" -[[package]] -name = "cmake" -version = "0.1.51" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.2" @@ -383,22 +297,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "crossbeam" version = "0.8.4" @@ -474,12 +372,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "either" version = "1.13.0" @@ -526,105 +418,18 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures-channel" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-sink" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "pin-utils", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" - [[package]] name = "glob" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "h2" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -661,112 +466,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "http" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" -dependencies = [ - "futures-util", - "http", - "hyper", - "hyper-util", - "log", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "socket2", - "tokio", - "tower", - "tower-service", - "tracing", -] - [[package]] name = "indexmap" version = "2.4.0" @@ -777,12 +476,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -804,15 +497,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.70" @@ -939,27 +623,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "metrics-exporter-prometheus" -version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" -dependencies = [ - "base64", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "indexmap", - "ipnet", - "metrics", - "metrics-util", - "quanta", - "thiserror", - "tokio", - "tracing", -] - [[package]] name = "metrics-util" version = "0.17.0" @@ -985,33 +648,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" -dependencies = [ - "hermit-abi", - "libc", - "wasi", - "windows-sys 0.52.0", -] - -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "nibble_vec" version = "0.1.0" @@ -1077,27 +713,12 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.36.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - [[package]] name = "ordered-float" version = "4.2.2" @@ -1113,38 +734,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.30" @@ -1275,21 +864,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rlimit" version = "0.10.1" @@ -1299,12 +873,6 @@ dependencies = [ "libc", ] -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -1324,62 +892,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.23.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" -dependencies = [ - "aws-lc-rs", - "log", - "once_cell", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" -dependencies = [ - "base64", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" - -[[package]] -name = "rustls-webpki" -version = "0.102.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" -dependencies = [ - "aws-lc-rs", - "ring", - "rustls-pki-types", - "untrusted", -] - [[package]] name = "rustversion" version = "1.0.17" @@ -1401,28 +913,21 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "scx_bpfland" version = "1.0.3" dependencies = [ "anyhow", "clap", + "crossbeam", "ctrlc", "libbpf-rs", "log", - "metrics", - "metrics-exporter-prometheus", "rlimit", + "scx_stats", + "scx_stats_derive", "scx_utils", + "serde", "simplelog", ] @@ -1440,6 +945,17 @@ dependencies = [ "syn", ] +[[package]] +name = "scx_stats_derive" +version = "1.0.3" +dependencies = [ + "proc-macro2", + "quote", + "scx_stats", + "serde_json", + "syn", +] + [[package]] name = "scx_utils" version = "1.0.3" @@ -1466,29 +982,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "1.0.23" @@ -1553,37 +1046,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - [[package]] name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -[[package]] -name = "socket2" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "sscanf" version = "0.4.2" @@ -1623,12 +1091,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.74" @@ -1742,109 +1204,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tokio" -version = "1.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "pin-project-lite", - "socket2", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls", - "rustls-pki-types", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "unicase" version = "2.7.0" @@ -1878,12 +1237,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - [[package]] name = "utf8parse" version = "0.2.2" @@ -1936,15 +1289,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2246,23 +1590,3 @@ dependencies = [ "quote", "syn", ] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/scheds/rust/scx_bpfland/Cargo.toml b/scheds/rust/scx_bpfland/Cargo.toml index 0259a34..12b413f 100644 --- a/scheds/rust/scx_bpfland/Cargo.toml +++ b/scheds/rust/scx_bpfland/Cargo.toml @@ -10,13 +10,15 @@ license = "GPL-2.0-only" anyhow = "1.0.65" ctrlc = { version = "3.1", features = ["termination"] } clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } +crossbeam = "0.8.4" libbpf-rs = "0.24.1" log = "0.4.17" +scx_stats = { path = "../../../rust/scx_stats", version = "1.0.3" } +scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.3" } scx_utils = { path = "../../../rust/scx_utils", version = "1.0.3" } +serde = { version = "1.0", features = ["derive"] } simplelog = "0.12" rlimit = "0.10.1" -metrics = "0.23.0" -metrics-exporter-prometheus = "0.15.0" [build-dependencies] scx_utils = { path = "../../../rust/scx_utils", version = "1.0.3" } diff --git a/scheds/rust/scx_bpfland/src/main.rs b/scheds/rust/scx_bpfland/src/main.rs index 547b693..40b7d32 100644 --- a/scheds/rust/scx_bpfland/src/main.rs +++ b/scheds/rust/scx_bpfland/src/main.rs @@ -10,6 +10,9 @@ pub use bpf_skel::*; pub mod bpf_intf; pub use bpf_intf::*; +mod stats; +use stats::Metrics; + use std::collections::HashMap; use std::ffi::c_int; use std::fs::File; @@ -24,11 +27,10 @@ use anyhow::bail; use anyhow::Context; use anyhow::Result; use clap::Parser; +use crossbeam::channel::RecvTimeoutError; use log::info; -use log::warn; -use metrics::{gauge, Gauge}; -use metrics_exporter_prometheus::PrometheusBuilder; +use log::warn; use rlimit::{getrlimit, setrlimit, Resource}; @@ -38,6 +40,8 @@ use libbpf_rs::skel::SkelBuilder; use libbpf_rs::OpenObject; use libbpf_rs::ProgramInput; +use scx_stats::prelude::*; + use scx_utils::build_id; use scx_utils::scx_ops_attach; use scx_utils::scx_ops_load; @@ -151,9 +155,7 @@ fn parse_cpumask(cpu_str: &str) -> Result { let cpus = get_primary_cpus(Powermode::Performance).unwrap(); Cpumask::from_str(&cpus_to_cpumask(&cpus)) } - "auto" => { - Cpumask::new() - } + "auto" => Cpumask::new(), _ if !cpu_str.is_empty() => Cpumask::from_str(&cpu_str.to_string()), _ => { let mut cpumask = Cpumask::new()?; @@ -243,9 +245,14 @@ struct Opts { #[clap(short = 't', long, default_value = "5000")] starvation_thresh_us: u64, - /// Enable the Prometheus endpoint for metrics on port 9000. - #[clap(short = 'p', long, action = clap::ArgAction::SetTrue)] - enable_prometheus: bool, + /// Enable stats monitoring with the specified interval. + #[clap(long)] + stats: Option, + + /// Run in stats monitoring mode with the specified interval. Scheduler + /// is not launched. + #[clap(long)] + monitor: Option, /// Enable BPF debugging via /sys/kernel/debug/tracing/trace_pipe. #[clap(short = 'd', long, action = clap::ArgAction::SetTrue)] @@ -258,44 +265,10 @@ struct Opts { /// Print scheduler version and exit. #[clap(short = 'V', long, action = clap::ArgAction::SetTrue)] version: bool, -} -struct Metrics { - nr_running: Gauge, - nr_interactive: Gauge, - nr_waiting: Gauge, - nvcsw_avg_thresh: Gauge, - nr_direct_dispatches: Gauge, - nr_prio_dispatches: Gauge, - nr_shared_dispatches: Gauge, -} - -impl Metrics { - fn new() -> Self { - Metrics { - nr_running: gauge!( - "nr_running", "info" => "Number of running tasks" - ), - nr_interactive: gauge!( - "nr_interactive", "info" => "Number of running interactive tasks" - ), - nr_waiting: gauge!( - "nr_waiting", "info" => "Average amount of tasks waiting to be dispatched" - ), - nvcsw_avg_thresh: gauge!( - "nvcsw_avg_thresh", "info" => "Average of voluntary context switches" - ), - nr_direct_dispatches: gauge!( - "nr_direct_dispatches", "info" => "Number of task direct dispatches" - ), - nr_prio_dispatches: gauge!( - "nr_prio_dispatches", "info" => "Number of interactive task dispatches" - ), - nr_shared_dispatches: gauge!( - "nr_shared_dispatches", "info" => "Number of regular task dispatches" - ), - } - } + /// Show descriptions for statistics. + #[clap(long)] + help_stats: bool, } fn is_smt_active() -> std::io::Result { @@ -312,9 +285,9 @@ struct Scheduler<'a> { skel: BpfSkel<'a>, struct_ops: Option, opts: &'a Opts, - metrics: Metrics, cpu_hotplug_cnt: u64, energy_profile: String, + stats_server: StatsServer<(), Metrics>, } impl<'a> Scheduler<'a> { @@ -363,10 +336,12 @@ impl<'a> Scheduler<'a> { // Initialize the primary scheduling domain (based on the --primary-domain option). let energy_profile = Self::read_energy_profile(); - if let Err(err) = Self::init_turbo_domain(&mut skel, &opts.primary_domain, &energy_profile) { + if let Err(err) = Self::init_turbo_domain(&mut skel, &opts.primary_domain, &energy_profile) + { warn!("failed to initialize turbo domain: error {}", err); } - if let Err(err) = Self::init_energy_domain(&mut skel, &opts.primary_domain, &energy_profile) { + if let Err(err) = Self::init_energy_domain(&mut skel, &opts.primary_domain, &energy_profile) + { warn!("failed to initialize primary domain: error {}", err); } @@ -381,22 +356,15 @@ impl<'a> Scheduler<'a> { // Attach the scheduler. let struct_ops = Some(scx_ops_attach!(skel, bpfland_ops)?); - - // Enable Prometheus metrics. - if opts.enable_prometheus { - info!("Enabling Prometheus endpoint: http://localhost:9000"); - PrometheusBuilder::new() - .install() - .expect("failed to install Prometheus recorder"); - } + let stats_server = StatsServer::new(stats::server_data()).launch()?; Ok(Self { skel, struct_ops, opts, - metrics: Metrics::new(), cpu_hotplug_cnt: 0, energy_profile, + stats_server, }) } @@ -423,12 +391,13 @@ impl<'a> Scheduler<'a> { } fn read_energy_profile() -> String { - let res = File::open("/sys/devices/system/cpu/cpufreq/policy0/energy_performance_preference") - .and_then(|mut file| { - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - Ok(contents.trim().to_string()) - }); + let res = + File::open("/sys/devices/system/cpu/cpufreq/policy0/energy_performance_preference") + .and_then(|mut file| { + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + Ok(contents.trim().to_string()) + }); res.unwrap_or_else(|_| "none".to_string()) } @@ -455,7 +424,11 @@ impl<'a> Scheduler<'a> { Ok(()) } - fn init_turbo_domain(skel: &mut BpfSkel<'_>, primary_domain: &Cpumask, energy_profile: &String) -> Result<()> { + fn init_turbo_domain( + skel: &mut BpfSkel<'_>, + primary_domain: &Cpumask, + energy_profile: &String, + ) -> Result<()> { let domain = if primary_domain.is_empty() { let cpus = match energy_profile.as_str() { "balance_power" => get_primary_cpus(Powermode::Turbo).unwrap_or(Vec::new()), @@ -489,12 +462,18 @@ impl<'a> Scheduler<'a> { Ok(()) } - fn init_energy_domain(skel: &mut BpfSkel<'_>, primary_domain: &Cpumask, energy_profile: &String) -> Result<()> { + fn init_energy_domain( + skel: &mut BpfSkel<'_>, + primary_domain: &Cpumask, + energy_profile: &String, + ) -> Result<()> { let domain = if primary_domain.is_empty() { let cpus = match energy_profile.as_str() { "power" => get_primary_cpus(Powermode::Powersave).unwrap_or(Vec::new()), "balance_power" => get_primary_cpus(Powermode::Performance).unwrap_or(Vec::new()), - "balance_performance" => get_primary_cpus(Powermode::Performance).unwrap_or(Vec::new()), + "balance_performance" => { + get_primary_cpus(Powermode::Performance).unwrap_or(Vec::new()) + } "performance" => get_primary_cpus(Powermode::Performance).unwrap_or(Vec::new()), &_ => Vec::new(), }; @@ -533,10 +512,18 @@ impl<'a> Scheduler<'a> { if energy_profile != self.energy_profile { self.energy_profile = energy_profile.clone(); - if let Err(err) = Self::init_turbo_domain(&mut self.skel, &self.opts.primary_domain, &energy_profile) { + if let Err(err) = Self::init_turbo_domain( + &mut self.skel, + &self.opts.primary_domain, + &energy_profile, + ) { warn!("failed to refresh turbo domain: error {}", err); } - if let Err(err) = Self::init_energy_domain(&mut self.skel, &self.opts.primary_domain, &energy_profile) { + if let Err(err) = Self::init_energy_domain( + &mut self.skel, + &self.opts.primary_domain, + &energy_profile, + ) { warn!("failed to refresh primary domain: error {}", err); } } @@ -663,42 +650,17 @@ impl<'a> Scheduler<'a> { self.cpu_hotplug_cnt = self.skel.maps.bss_data.cpu_hotplug_cnt; } - fn update_stats(&mut self) { - let nr_cpus = self.skel.maps.bss_data.nr_online_cpus; - let nr_running = self.skel.maps.bss_data.nr_running; - let nr_interactive = self.skel.maps.bss_data.nr_interactive; - let nr_waiting = self.skel.maps.bss_data.nr_waiting; - let nvcsw_avg_thresh = self.skel.maps.bss_data.nvcsw_avg_thresh; - let nr_direct_dispatches = self.skel.maps.bss_data.nr_direct_dispatches; - let nr_prio_dispatches = self.skel.maps.bss_data.nr_prio_dispatches; - let nr_shared_dispatches = self.skel.maps.bss_data.nr_shared_dispatches; - - // Update Prometheus statistics. - self.metrics.nr_running.set(nr_running as f64); - self.metrics.nr_interactive.set(nr_interactive as f64); - self.metrics.nr_waiting.set(nr_waiting as f64); - self.metrics.nvcsw_avg_thresh.set(nvcsw_avg_thresh as f64); - self.metrics - .nr_direct_dispatches - .set(nr_direct_dispatches as f64); - self.metrics - .nr_prio_dispatches - .set(nr_prio_dispatches as f64); - self.metrics - .nr_shared_dispatches - .set(nr_shared_dispatches as f64); - - // Log scheduling statistics. - info!("[{}] tasks -> run: {:>2}/{:<2} int: {:<2} wait: {:<4} | nvcsw: {:<4} | dispatch -> dir: {:<5} prio: {:<5} shr: {:<5}", - SCHEDULER_NAME, - nr_running, - nr_cpus, - nr_interactive, - nr_waiting, - nvcsw_avg_thresh, - nr_direct_dispatches, - nr_prio_dispatches, - nr_shared_dispatches); + fn get_metrics(&self) -> Metrics { + Metrics { + nr_running: self.skel.maps.bss_data.nr_running, + nr_cpus: self.skel.maps.bss_data.nr_online_cpus, + nr_interactive: self.skel.maps.bss_data.nr_interactive, + nr_waiting: self.skel.maps.bss_data.nr_waiting, + nvcsw_avg_thresh: self.skel.maps.bss_data.nvcsw_avg_thresh, + nr_direct_dispatches: self.skel.maps.bss_data.nr_direct_dispatches, + nr_prio_dispatches: self.skel.maps.bss_data.nr_prio_dispatches, + nr_shared_dispatches: self.skel.maps.bss_data.nr_shared_dispatches, + } } pub fn exited(&mut self) -> bool { @@ -706,13 +668,16 @@ impl<'a> Scheduler<'a> { } fn run(&mut self, shutdown: Arc) -> Result { + let (res_ch, req_ch) = self.stats_server.channels(); while !shutdown.load(Ordering::Relaxed) && !self.exited() { self.refresh_cache_domains(); self.refresh_sched_domain(); - self.update_stats(); - std::thread::sleep(Duration::from_millis(1000)); + match req_ch.recv_timeout(Duration::from_secs(1)) { + Ok(()) => res_ch.send(self.get_metrics())?, + Err(RecvTimeoutError::Timeout) => {} + Err(e) => Err(e)?, + } } - self.update_stats(); self.struct_ops.take(); uei_report!(&self.skel, uei) @@ -733,6 +698,11 @@ fn main() -> Result<()> { return Ok(()); } + if opts.help_stats { + stats::server_data().describe_meta(&mut std::io::stdout(), None)?; + return Ok(()); + } + let loglevel = simplelog::LevelFilter::Info; let mut lcfg = simplelog::ConfigBuilder::new(); @@ -754,6 +724,17 @@ fn main() -> Result<()> { }) .context("Error setting Ctrl-C handler")?; + if let Some(intv) = opts.monitor.or(opts.stats) { + let shutdown_copy = shutdown.clone(); + let jh = std::thread::spawn(move || { + stats::monitor(Duration::from_secs_f64(intv), shutdown_copy).unwrap() + }); + if opts.monitor.is_some() { + let _ = jh.join(); + return Ok(()); + } + } + let mut open_object = MaybeUninit::uninit(); loop { let mut sched = Scheduler::init(&opts, &mut open_object)?; diff --git a/scheds/rust/scx_bpfland/src/stats.rs b/scheds/rust/scx_bpfland/src/stats.rs new file mode 100644 index 0000000..31d55c8 --- /dev/null +++ b/scheds/rust/scx_bpfland/src/stats.rs @@ -0,0 +1,74 @@ +use anyhow::Result; +use scx_stats::prelude::*; +use scx_stats_derive::Stats; +use serde::Deserialize; +use serde::Serialize; +use std::io::Write; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)] +#[stat(top)] +pub struct Metrics { + #[stat(desc = "Number of running tasks")] + pub nr_running: u64, + #[stat(desc = "Number of online CPUs")] + pub nr_cpus: u64, + #[stat(desc = "Number of running interactive tasks")] + pub nr_interactive: u64, + #[stat(desc = "Average amount of tasks waiting to be dispatched")] + pub nr_waiting: u64, + #[stat(desc = "Average of voluntary context switches")] + pub nvcsw_avg_thresh: u64, + #[stat(desc = "Number of task direct dispatches")] + pub nr_direct_dispatches: u64, + #[stat(desc = "Number of interactive task dispatches")] + pub nr_prio_dispatches: u64, + #[stat(desc = "Number of regular task dispatches")] + pub nr_shared_dispatches: u64, +} + +impl Metrics { + fn format(&self, w: &mut W) -> Result<()> { + writeln!( + w, + "[{}] tasks -> run: {:>2}/{:<2} int: {:<2} wait: {:<4} | nvcsw: {:<4} | dispatch -> dir: {:<5} prio: {:<5} shr: {:<5}", + crate::SCHEDULER_NAME, + self.nr_running, + self.nr_cpus, + self.nr_interactive, + self.nr_waiting, + self.nvcsw_avg_thresh, + self.nr_direct_dispatches, + self.nr_prio_dispatches, + self.nr_shared_dispatches + )?; + Ok(()) + } +} + +pub fn server_data() -> StatsServerData<(), Metrics> { + let open: Box> = Box::new(move |(_req_ch, _res_ch)| { + let read: Box> = Box::new(move |_args, (req_ch, res_ch)| { + req_ch.send(())?; + let metrics = res_ch.recv()?; + metrics.to_json() + }); + Ok(read) + }); + + StatsServerData::new() + .add_meta(Metrics::meta()) + .add_ops("top", StatsOps { open, close: None }) +} + +pub fn monitor(intv: Duration, shutdown: Arc) -> Result<()> { + scx_utils::monitor_stats::( + &vec![], + intv, + || shutdown.load(Ordering::Relaxed), + |metrics| metrics.format(&mut std::io::stdout()), + ) +}