Fetch and build bpftool by default

This pairs with the new default behavior to fetch and build libbpf
and is mostly being used so we can use the latest bpftool and libbpf.

.github/workflows/build-scheds.yml

@@ -29,11 +29,6 @@ jobs:
       - run: sudo ln -sf /usr/bin/clang-17 /usr/bin/clang
       - run: sudo ln -sf /usr/bin/llvm-strip-17 /usr/bin/llvm-strip
 
-      # bpftool
-      - run: git clone --recurse-submodules --branch v7.3.0 https://github.com/libbpf/bpftool.git
-      - run: make -j -C bpftool/src
-      - run: sudo make -j -C bpftool/src install
-
       - uses: actions/checkout@v4
 
       # meson

README.md

@@ -201,7 +201,7 @@ repo](https://mesonbuild.com/Quick-guide.html#installation-from-source) and call
   new in 1.3). It's preferred to link statically against the source from the libbpf git repo, which is cloned during setup.
 - Rust toolchain: >=1.72
 - `libelf`, `libz`, `libzstd` if linking against staic `libbpf.a`
-- `bpftool` (usually available in `linux-tools-common`)
+- `bpftool` By default this is cloned and built as part of the default build process. Alternatively it's usually available in `linux-tools-common`.
 
 
 ### Setting Up and Building
@@ -219,7 +219,7 @@ $ meson compile -C build
 $ meson install -C build
 ```
 
-Note: `meson setup` will also clone the libbpf repo and `meson compile` will build it.
+Notes: `meson setup` will also clone both libbpf and bpftool repos and `meson compile` will build them both.
 
 #### Dynamic linking against libbpf
 ```
@@ -229,6 +229,16 @@ $ meson compile -C build
 $ meson install -C build
 ```
 
+#### Using a different bpftool
+This will check the system for an installed bpftool
+```
+$ meson setup build --prefix ~ -D bpftool=disabled
+```
+Using a custom built bpftool
+```
+$ meson setup build --prefix ~ -D bpftool=/path/to/bpftool
+```
+
 Note that `meson compile` step is not strictly necessary as `install`
 implies `compile`. The above also will build debug binaries with
 optimizations turned off, which is useful for development but they aren't
@@ -287,7 +297,7 @@ override some of the toolchains and dependencies - e.g. to directly use
 options can be used in such cases.
 
 - `bpf_clang`: `clang` to use when compiling `.bpf.c`
-- `bpftool`: `bpftool` to use when generating `.bpf.skel.h`
+- `bpftool`: `bpftool` to use when generating `.bpf.skel.h`. Set this to "disabled" to check the system for an already installed bpftool
 - `libbpf_a`: Static `libbpf.a` to use. Set this to "disabled" to link libbpf dynamically
 - `libbpf_h`: `libbpf` header directories, only meaningful with `libbpf_a` option
 - `cargo`: `cargo` to use when building rust sub-projects

meson-scripts/bpftool_dummy.c

@@ -0,0 +1,4 @@
+// This is just used as a dummy target file for meson
+int main(void) {
+    return 0;
+}

meson-scripts/build_bpftool

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+out=$("$1" 'map(select(.["file"] | contains ("cc_cflags_probe.c"))) | first | .["command"]' < compile_commands.json)
+out=${out#\"}
+out=${out%\"}
+args=($out)
+
+idx=0
+cc=${args[idx]}
+if [ "$cc" = "ccache" ]; then
+    idx=$((idx+1))
+    cc="$cc ${args[idx]}"
+fi
+
+declare -a cflags=()
+
+for arg in ${args[@]:(idx+1)}; do
+    case $arg in
+	-I*|-M*|-o|-c) ;;
+	-*) cflags+="$arg ";;
+    esac
+done
+
+cd $3
+make_out=$(env CC="$cc" CFLAGS="$cflags" "$2" -j"$4")
+exit $?

meson-scripts/fetch_bpftool

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $1
+rm -rf bpftool
+git clone --depth=1 https://github.com/libbpf/bpftool.git
+cd bpftool
+git fetch --depth=1 origin $2
+git checkout $2
+git submodule update --init --recursive

meson.build

@@ -13,7 +13,6 @@ cc = meson.get_compiler('c')
 enable_rust = get_option('enable_rust')
 
 bpf_clang = find_program(get_option('bpf_clang'))
-bpftool = find_program(get_option('bpftool'))
 
 if enable_rust
   cargo = find_program(get_option('cargo'))
@@ -32,6 +31,10 @@ fetch_libbpf = find_program(join_paths(meson.current_source_dir(),
                                       'meson-scripts/fetch_libbpf'))
 build_libbpf = find_program(join_paths(meson.current_source_dir(),
                                       'meson-scripts/build_libbpf'))
+fetch_bpftool = find_program(join_paths(meson.current_source_dir(),
+                                      'meson-scripts/fetch_bpftool'))
+build_bpftool = find_program(join_paths(meson.current_source_dir(),
+                                      'meson-scripts/build_bpftool'))
 if enable_rust
   cargo_fetch = find_program(join_paths(meson.current_source_dir(),
                                       'meson-scripts/cargo_fetch'))
@@ -48,6 +51,25 @@ elif bpf_clang_maj < 17
           .format(bpf_clang.full_path(), bpf_clang_ver))
 endif
 
+# These are for building libbpf and/or bpftool
+
+if meson.get_compiler('c').get_id() == 'gcc'
+  extra_args = ['-Wno-sign-compare']
+else
+  extra_args = []
+endif
+
+executable('cc_cflags_probe', 'meson-scripts/cc_cflags_probe.c', install: false, pie: true, c_args : extra_args)
+
+jq = find_program('jq')
+make = find_program('make')
+nproc = find_program('nproc')
+
+make_jobs = 1
+if nproc.found()
+  make_jobs = run_command(nproc, check: true).stdout().to_int()
+endif
+
 libbpf_path = '@0@/libbpf/src'.format(meson.current_build_dir())
 libbpf_a = '@0@/libbpf.a'.format(libbpf_path)
 should_build_libbpf = true
@@ -70,10 +92,6 @@ endif
 # then read the compiler and args from meson's compile_commands.json
 # and re-set them when we build libbpf with make
 if should_build_libbpf
-  jq = find_program('jq')
-  make = find_program('make')
-  nproc = find_program('nproc')
-
   if not jq.found() or not make.found()
     error('To build the libbpf library "make" and "jq" are required')
   endif
@@ -85,9 +103,6 @@ if should_build_libbpf
   libbpf_commit = '4f875865b772c4f534bc0a665bbd988193825bd4'
   run_command(fetch_libbpf, meson.current_build_dir(), libbpf_commit, check: true)
 
-  # setup the build target
-  executable('cc_cflags_probe', 'meson-scripts/cc_cflags_probe.c', install: false, pie: true)
-
   make_jobs = 1
   if nproc.found()
     make_jobs = run_command(nproc, check: true).stdout().to_int()
@@ -121,6 +136,40 @@ else
   endif
 endif
 
+bpftool_path = '@0@/bpftool/src'.format(meson.current_build_dir())
+should_build_bpftool = true
+bpftool_exe_path = '@0@/bpftool'.format(bpftool_path)
+
+if get_option('bpftool') == 'disabled'
+  bpftool = find_program('bpftool')
+  should_build_bpftool = false
+  bpftool_exe_path = bpftool.full_path()
+elif get_option('bpftool') != ''
+  bpftool = find_program(get_option('bpftool'))
+  should_build_bpftool = false
+  bpftool_exe_path = bpftool.full_path()
+endif
+
+if should_build_bpftool
+  message('Fetching bpftool repo')
+  bpftool_commit = '8328f373c0ef27fda14f12e67f1d6ed882dd2e81'
+  run_command(fetch_bpftool, meson.current_build_dir(), bpftool_commit, check: true)
+  
+  bpftool_target = custom_target('bpftool_target',
+              output: '@PLAINNAME@.__PHONY__',
+              input: 'meson-scripts/bpftool_dummy.c',
+              command: [build_bpftool, jq, make, bpftool_path, '@0@'.format(make_jobs)],
+              build_by_default: true)
+else
+  # this is a noop when we're not building bpftool ourselves
+  bpftool_target = custom_target('bpftool_target',
+              output: '@PLAINNAME@.__PHONY__',
+              input: 'meson-scripts/bpftool_dummy.c',
+              command: ['echo'],
+              build_by_default: true)
+endif
+# end libbpf/bpftool stuff
+
 #
 # Determine bpf_base_cflags which will be used to compile all .bpf.o files.
 # Note that "-target bpf" is not included to accommodate
@@ -177,8 +226,8 @@ gen_bpf_o = generator(bpf_clang,
                                   '-c', '@INPUT@', '-o', '@OUTPUT@'])
 gen_bpf_skel = generator(bpftool_build_skel,
                          output: ['@BASENAME@.skel.h','@BASENAME@.subskel.h' ],
-                         depends: [libbpf],
-                         arguments: [bpftool.full_path(), '@INPUT@', '@OUTPUT0@', '@OUTPUT1@'])
+                         depends: [libbpf, bpftool_target],
+                         arguments: [bpftool_exe_path, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@'])
 
 #
 # For rust sub-projects.

meson.options

@@ -1,7 +1,7 @@
 option('bpf_clang', type: 'string', value: 'clang',
        description: 'clang to use when compiling .bpf.c')
-option('bpftool', type: 'string', value: 'bpftool',
-       description: 'bpftool to use when generating .bpf.skel.h')
+option('bpftool', type: 'string',
+       description: 'bpftool to use when generating .bpf.skel.h. By default, bpftool is automatically downloaded and built during setup. To use an existing bpftool binary, point this to the bpftool path or set it to "disabled" to have meson try to find it automatically')
 option('libbpf_a', type: 'string',
        description: 'By default, libbpf is automatically downloaded and built during setup. To use an existing static library, point this to the libbpf.a file or set it to "disabled" to use dynamic linking')
 option('libbpf_h', type: 'array',

rust/scx_rustland_core/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_rustland_core',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)

rust/scx_utils/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_utils',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)

scheds/rust/scx_layered/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_layered',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)

scheds/rust/scx_rlfifo/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_rlfifo',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)

scheds/rust/scx_rustland/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_rustland',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)

scheds/rust/scx_rusty/meson.build

@@ -4,5 +4,5 @@ custom_target('scx_rusty',
               command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@',
                         cargo_build_args],
               env: cargo_env,
-              depends: [libbpf],
+              depends: [libbpf, bpftool_target],
               build_by_default: true)