Merge pull request #632 from likewhatevs/split-integration-test

split integration test execution

.github/actions/install-deps-action/action.yml

@@ -16,7 +16,7 @@ runs:
     ### DOWNLOAD AND INSTALL DEPENDENCIES ###
 
     # Download dependencies packaged by Ubuntu
-    - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd
+    - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd libseccomp-dev libcap-ng-dev
       shell: bash
 
     # clang 17

.github/workflows/caching-build.yml

@@ -25,11 +25,22 @@ jobs:
         with:
           path: |
             linux
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
         uses: ./.github/actions/install-deps-action
 
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' && steps.cache-kernel.outputs.cache-hit != 'true' }}
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # cache bzImage alone for rust tests (disk space limit workaround)
       - name: Cache bzImage
         id: cache-bzImage
@@ -37,7 +48,7 @@ jobs:
         with:
           path: |
             linux/arch/x86/boot/bzImage
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
         name: Clone Kernel
@@ -64,10 +75,23 @@ jobs:
   integration-test:
     runs-on: ubuntu-22.04
     needs: build-kernel
+    strategy:
+          matrix:
+            scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install-deps-action
-      # cache rust deps, invalidate when kernel commit changes
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # get latest head commit of sched_ext for-next
       - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
 
@@ -78,7 +102,7 @@ jobs:
         with:
           path: |
             linux
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       # need to re-run job when kernel head changes between build and test running.
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
@@ -92,25 +116,42 @@ jobs:
 
       # The actual build:
       - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true
-      - run: meson compile -C build
+      - run: meson compile -C build ${{ matrix.scheduler }}
 
       # Print CPU model before running the tests (this can be useful for
       # debugging purposes)
       - run: grep 'model name' /proc/cpuinfo | head -1
 
       # Test schedulers
-      - run: meson compile -C build test_sched
+      - run: meson compile -C build test_sched_${{ matrix.scheduler }}
       # Stress schedulers
-      - run: meson compile -C build stress_tests
-      - run: meson compile -C build veristat
+      - uses: cytopia/shell-command-retry-action@v0.1.2
+        name: stress test
+        with:
+          retries: 3
+          command: meson compile -C build stress_tests_${{ matrix.scheduler }}
+      - run: meson compile -C build veristat_${{ matrix.scheduler }}
 
-  rust-test:
+  rust-test-core:
     runs-on: ubuntu-22.04
     needs: build-kernel
+    strategy:
+      matrix:
+        component: [scx_loader, scx_rustland_core, scx_stats, scx_utils]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install-deps-action
-      # cache rust deps, invalidate when kernel commit changes
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # get latest head commit of sched_ext for-next
       - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
       # cache bzImage alone for rust tests
@@ -120,7 +161,7 @@ jobs:
         with:
           path: |
             linux/arch/x86/boot/bzImage
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       # need to re-run job when kernel head changes between build and test running.
       - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }}
@@ -129,9 +170,55 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          shared-key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
           workspaces: rust
+          key: ${{ matrix.component }}
           prefix-key: "1"
-      - run: cargo build  --manifest-path rust/Cargo.toml
-      - run: cargo test --manifest-path rust/Cargo.toml --no-run
-      - run: vng -v --memory 10G --cpu 8 --force-9p -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path rust/Cargo.toml
+      - run: cargo build  --manifest-path rust/${{ matrix.component }}/Cargo.toml
+      - run: cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml --no-run
+      - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml
+
+  rust-test-schedulers:
+    runs-on: ubuntu-22.04
+    needs: build-kernel
+    strategy:
+      matrix:
+        scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/install-deps-action
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
+      # get latest head commit of sched_ext for-next
+      - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
+      # cache bzImage alone for rust tests
+      - name: Cache bzImage
+        id: cache-bzImage
+        uses: actions/cache@v4
+        with:
+          path: |
+            linux/arch/x86/boot/bzImage
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
+
+      # need to re-run job when kernel head changes between build and test running.
+      - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }}
+        name: exit if cache stale
+        run: exit -1
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: scheds/rust
+          key: ${{ matrix.scheduler }}
+          prefix-key: "1"
+      - run: cargo build  --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml
+      - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run
+      - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml
+

meson-scripts/run_stress_tests

@@ -48,7 +48,7 @@ def run_stress_test(
     timeout_sec = int(config.get("timeout_sec"))
     if vng_path:
         vm_input = f"{stress_cmd} & timeout --foreground --preserve-status {timeout_sec} {sched_cmd}"
-        cmd = [vng_path, "--user", "root", "--force-9p", "-v", "--", vm_input]
+        cmd = [vng_path, "--user", "root", "-v", "--", vm_input]
         err = sys.stderr if output == "-" else open(output, "w")
         out = sys.stdout if output == "-" else err
         proc = subprocess.Popen(
@@ -73,6 +73,9 @@ def stress_tests(args: Namespace) -> None:
     for test_name in configs.sections():
         if args.stress_test and test_name != args.stress_test:
             continue
+        if args.sched and configs[test_name].get('sched') != args.sched:
+            continue
+        print(f"Running stress test: {test_name}")
         return_codes[test_name] = run_stress_test(
             configs[test_name],
             args.build_dir,
@@ -106,6 +109,9 @@ if __name__ == "__main__":
         '-v', '--verbose', action='store_true', help='Verbose output')
     parser.add_argument(
         '--vng', action='store_true', default=True, help='Run in vng')
+    parser.add_argument(
+        '--sched', default='', help='Scheduler to test (default: all)'
+    )
 
     args = parser.parse_args()
     if args.verbose:

meson-scripts/test_sched

@@ -18,13 +18,27 @@ GUEST_TIMEOUT=60
 #
 declare -A SCHEDS
 
-SCHEDS["scx_simple"]=""
-SCHEDS["scx_central"]=""
-SCHEDS["scx_nest"]=""
-SCHEDS["scx_rusty"]=""
-SCHEDS["scx_rustland"]=""
-SCHEDS["scx_bpfland"]=""
-SCHEDS["scx_layered"]="--run-example"
+# enable running tests on individual schedulers
+if [ $# -ge 2 ] ; then
+    SCHEDS[$2]=""
+else
+    SCHEDS["scx_simple"]=""
+    SCHEDS["scx_central"]=""
+    SCHEDS["scx_nest"]=""
+    SCHEDS["scx_rusty"]=""
+    SCHEDS["scx_rustland"]=""
+    SCHEDS["scx_bpfland"]=""
+    SCHEDS["scx_layered"]="--run-example"
+fi
+
+if [[ -v SCHEDS["scx_layered"] ]] ; then
+    SCHEDS["scx_layered"]="--run-example"
+fi
+
+printf "testing scheds:\n"
+for i in "${!SCHEDS[@]}"; do
+    printf "%s=%s\n" "$i" "${SCHEDS[$i]}";
+done
 
 if [ ! -x `which vng` ]; then
     echo "vng not found, please install virtme-ng to enable testing"
@@ -48,7 +62,7 @@ for sched in ${!SCHEDS[@]}; do
 
     rm -f /tmp/output
     timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v -r ${kernel} -- \
+        vng -m 2G -v -r ${kernel} -- \
             "timeout --foreground --preserve-status ${TEST_TIMEOUT} ${sched_path} ${args}" \
                 2> >(tee /tmp/output) </dev/null
         grep -v " Speculative Return Stack Overflow" /tmp/output | \

meson-scripts/veristat

@@ -20,7 +20,7 @@ if [ -n "${SCHED}" ]; then
   echo "Running veristat on ${BPF_PATH}"
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --user root -r ${KERNEL} -- \
+      vng -m 2G -v --user root -r ${KERNEL} -- \
         sudo veristat ${BPF_PATH}
     exit $?
   else
@@ -32,7 +32,7 @@ fi
 for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --user root -r ${KERNEL} -- \
+      vng -m 2G -v --user root -r ${KERNEL} -- \
         sudo veristat ${BPF_PATH}
   else
     echo "$BPF_PATH"

meson-scripts/veristat_diff

@@ -21,12 +21,12 @@ if [ -n "${SCHED}" ]; then
   echo "Running veristat on ${BPF_PATH}"
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+      vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
         veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \
 	  || echo "failed to verify ${SCHED}: ${BPF_PATH}"
     if [ -n "${DIFF_DIR}" ]; then
       timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+        vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
           veristat -C "${DIFF_DIR}/${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \
 	    -e file,prog,verdict,insns \
 	    || echo "failed to compare ${SCHED}: ${BPF_PATH}"
@@ -47,12 +47,12 @@ for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do
   SCHED=$(echo "${BPF_PATH}" | sed 's/.*\/rust\///g' | sed 's/\/.*//g')
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+      vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
         veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \
           || echo "failed to verify ${SCHED}: ${BPF_PATH}"
     if [ -n "${DIFF_DIR}" ]; then
       timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+        vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
           veristat -C "${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \
 	    -e file,prog,verdict,insns \
             || echo "failed to compare ${SCHED}: ${BPF_PATH}"

meson.build

@@ -330,8 +330,21 @@ endif
 
 run_target('test_sched', command: [test_sched, kernel])
 
+SCHEDULERS = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo',
+               'scx_rusty',
+               'scx_layered'] # scx_mitosis temporarily excluded                        
+
+foreach s : SCHEDULERS
+  run_target('test_sched_'+s, command: [test_sched, kernel, s])
+endforeach
+
 run_target('veristat', command: [run_veristat, meson.current_build_dir(),
                                  get_option('veristat_scheduler'), get_option('kernel')])
+foreach s : SCHEDULERS
+  run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(),
+                                 get_option('veristat_scheduler'), get_option('kernel'), s])
+endforeach
+
 run_target('veristat_diff', command: [run_veristat_diff, meson.current_build_dir(),
                                  get_option('veristat_scheduler'), get_option('kernel'),
                                  get_option('veristat_diff_dir')])
@@ -343,6 +356,10 @@ subdir('rust')
 if enable_stress
   run_target('stress_tests', command: [run_stress_tests, '-k', kernel, '-b',
     meson.current_build_dir()])
+foreach s : SCHEDULERS
+  run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b',
+    meson.current_build_dir(), '--sched', s])
+endforeach
 endif
 
 systemd = dependency('systemd', required: get_option('systemd'))

scheds/rust/meson.build

@@ -12,12 +12,7 @@ custom_target('rust_scheds',
               build_by_default: true,
               build_always_stale: true)
 
-# per-scheduler targets
-rust_scheds = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo',
-               'scx_rusty',
-               'scx_layered'] # scx_mitosis temporarily excluded
-
-foreach sched: rust_scheds
+foreach sched: SCHEDULERS
   custom_target(sched,
                 output: sched + '@PLAINNAME@.__PHONY__',
                 input: 'Cargo.toml',