From b535b8f82e223d96691ff59efa02e36e374498d9 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 1 Aug 2022 11:22:18 -0700 Subject: [PATCH] vmtest: don't use BusyBox We don't specifically need BusyBox; we just need a reasonable Linux userspace, which we can assume is already available on the host, whether it's coreutils+util-linux, BusyBox, or something else. Signed-off-by: Omar Sandoval --- .github/workflows/ci.yml | 2 +- setup.py | 4 +- vmtest/README.rst | 9 ++- vmtest/vm.py | 116 +++++++++++++++++++-------------------- 4 files changed, 64 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 111d4c80..c49a157b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install busybox-static libelf-dev libdw-dev qemu-kvm zstd ${{ matrix.cc == 'clang' && 'libomp-$(clang --version | sed -rn "s/.*clang version ([0-9]+).*/\\1/p")-dev' || '' }} + sudo apt-get install libelf-dev libdw-dev qemu-kvm zstd ${{ matrix.cc == 'clang' && 'libomp-$(clang --version | sed -rn "s/.*clang version ([0-9]+).*/\\1/p")-dev' || '' }} pip install pyroute2 pre-commit - name: Generate version.py run: python setup.py --version diff --git a/setup.py b/setup.py index 32005527..60b435f8 100755 --- a/setup.py +++ b/setup.py @@ -258,10 +258,10 @@ class test(Command): set -e export DRGN_TEST_KMOD={shlex.quote(str(kmod))} -if "$BUSYBOX" [ -e /proc/vmcore ]; then +if [ -e /proc/vmcore ]; then "$PYTHON" -Bm unittest discover -t . -s tests/linux_kernel/vmcore {"-v" if self.verbose else ""} else - "$BUSYBOX" insmod "$DRGN_TEST_KMOD" + insmod "$DRGN_TEST_KMOD" DRGN_RUN_LINUX_KERNEL_TESTS=1 "$PYTHON" -Bm \ unittest discover -t . -s tests/linux_kernel {"-v" if self.verbose else ""} "$PYTHON" vmtest/enter_kdump.py diff --git a/vmtest/README.rst b/vmtest/README.rst index 2fda09ba..38ea3332 100644 --- a/vmtest/README.rst +++ b/vmtest/README.rst @@ -4,8 +4,8 @@ drgn VM Testing drgn has a significant amount of code (both core and in helpers) which is dependent on the Linux kernel version. This code is tested on multiple Linux kernel versions in a virtual machine. These tests can be run on all supported -kernels with ``python3 setup.py test -K``. This requires QEMU, BusyBox, and -zstd to be installed. +kernels with ``python3 setup.py test -K``. This requires QEMU and zstd to be +installed. Tests can also be run on specific kernels with ``-k``. This takes a comma-separated list of kernels which are wildcard patterns (e.g., ``5.6.*``) @@ -25,9 +25,8 @@ safety. To support modifications, the guest uses `OverlayFS overlay a read-write tmpfs over the VirtFS root. It also mounts the kernel modules and vmlinux via VirtFS. -The guest runs a `BusyBox `_ shell script as init -which sets up the system and filesystem hierarchy, runs a command, and returns -the exit status via `virtio-serial +The guest runs an init shell script which sets up the system and filesystem +hierarchy, runs a command, and returns the exit status via `virtio-serial `_. This infrastructure is all generic. The drgn-specific parts are: diff --git a/vmtest/vm.py b/vmtest/vm.py index 43238d1e..5fbd7add 100644 --- a/vmtest/vm.py +++ b/vmtest/vm.py @@ -1,12 +1,10 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # SPDX-License-Identifier: GPL-3.0-or-later -import errno import os from pathlib import Path import re import shlex -import shutil import socket import subprocess import sys @@ -16,110 +14,114 @@ from util import nproc, out_of_date _9PFS_MSIZE = 1024 * 1024 -# Script run as init in the virtual machine. This only depends on busybox. We -# don't assume that any regular commands are built in (not even echo or test), -# so we always explicitly run busybox. -_INIT_TEMPLATE = r"""#!{busybox} sh +# Script run as init in the virtual machine. +_INIT_TEMPLATE = r"""#!/bin/sh + +# Having /proc from the host visible in the guest can confuse some commands. In +# particular, if BusyBox is configured with FEATURE_SH_STANDALONE, then busybox +# sh executes BusyBox applets using /proc/self/exe. So, before doing anything +# else, mount /proc (using the fully qualified executable path so that BusyBox +# doesn't use /proc/self/exe). +/bin/mount -t proc -o nosuid,nodev,noexec proc /proc set -eu -export BUSYBOX={busybox} +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PYTHON={python} {kdump_needs_nosmp} -trap '"$BUSYBOX" poweroff -f' EXIT +trap 'poweroff -f' EXIT umask 022 HOSTNAME=vmtest VPORT_NAME=com.osandov.vmtest.0 -RELEASE=$("$BUSYBOX" uname -r) +RELEASE=$(uname -r) # Set up overlayfs on the temporary directory containing this script. -mnt=$("$BUSYBOX" dirname "$0") -"$BUSYBOX" mount -t tmpfs tmpfs "$mnt" -"$BUSYBOX" mkdir "$mnt/upper" "$mnt/work" "$mnt/merged" +mnt=$(dirname "$0") +mount -t tmpfs tmpfs "$mnt" +mkdir "$mnt/upper" "$mnt/work" "$mnt/merged" -"$BUSYBOX" mkdir "$mnt/upper/dev" "$mnt/upper/etc" "$mnt/upper/mnt" -"$BUSYBOX" mkdir -m 555 "$mnt/upper/proc" "$mnt/upper/sys" -"$BUSYBOX" mkdir -m 1777 "$mnt/upper/tmp" +mkdir "$mnt/upper/dev" "$mnt/upper/etc" "$mnt/upper/mnt" +mkdir -m 555 "$mnt/upper/proc" "$mnt/upper/sys" +mkdir -m 1777 "$mnt/upper/tmp" -# Create configuration files. -"$BUSYBOX" cat << EOF > "$mnt/upper/etc/hosts" -127.0.0.1 localhost -::1 localhost -127.0.1.1 $HOSTNAME.localdomain $HOSTNAME -EOF -: > "$mnt/upper/etc/resolv.conf" +mount -t overlay -o lowerdir=/,upperdir="$mnt/upper",workdir="$mnt/work" overlay "$mnt/merged" -"$BUSYBOX" mount -t overlay -o lowerdir=/,upperdir="$mnt/upper",workdir="$mnt/work" overlay "$mnt/merged" -"$BUSYBOX" pivot_root "$mnt/merged" "$mnt/merged/mnt" -cd / -"$BUSYBOX" umount -l /mnt - -# Mount additional filesystems. -"$BUSYBOX" mount -t devtmpfs -o nosuid,noexec dev /dev -"$BUSYBOX" mkdir /dev/shm -"$BUSYBOX" mount -t tmpfs -o nosuid,nodev tmpfs /dev/shm -"$BUSYBOX" mount -t proc -o nosuid,nodev,noexec proc /proc -"$BUSYBOX" mount -t sysfs -o nosuid,nodev,noexec sys /sys +# Mount core filesystems. +mount -t devtmpfs -o nosuid,noexec dev "$mnt/merged/dev" +mkdir "$mnt/merged/dev/shm" +mount -t tmpfs -o nosuid,nodev tmpfs "$mnt/merged/dev/shm" +mount -t proc -o nosuid,nodev,noexec proc "$mnt/merged/proc" +mount -t sysfs -o nosuid,nodev,noexec sys "$mnt/merged/sys" # cgroup2 was added in Linux v4.5. -"$BUSYBOX" mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 /sys/fs/cgroup || "$BUSYBOX" true +mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 "$mnt/merged/sys/fs/cgroup" || true # Ideally we'd just be able to create an opaque directory for /tmp on the upper # layer. However, before Linux kernel commit 51f7e52dc943 ("ovl: share inode # for hard link") (in v4.8), overlayfs doesn't handle hard links correctly, # which breaks some tests. -"$BUSYBOX" mount -t tmpfs -o nosuid,nodev tmpfs /tmp +mount -t tmpfs -o nosuid,nodev tmpfs "$mnt/merged/tmp" + +# Pivot into the new root. +pivot_root "$mnt/merged" "$mnt/merged/mnt" +cd / +umount -l /mnt # Load kernel modules. -"$BUSYBOX" mkdir -p "/lib/modules/$RELEASE" -"$BUSYBOX" mount -t 9p -o trans=virtio,cache=loose,ro,msize={_9PFS_MSIZE} modules "/lib/modules/$RELEASE" +mkdir -p "/lib/modules/$RELEASE" +mount -t 9p -o trans=virtio,cache=loose,ro,msize={_9PFS_MSIZE} modules "/lib/modules/$RELEASE" for module in configs rng_core virtio_rng; do - "$BUSYBOX" modprobe "$module" + modprobe "$module" done # Create static device nodes. -"$BUSYBOX" grep -v '^#' "/lib/modules/$RELEASE/modules.devname" | +grep -v '^#' "/lib/modules/$RELEASE/modules.devname" | while read -r module name node; do name="/dev/$name" dev=${{node#?}} major=${{dev%%:*}} minor=${{dev##*:}} type=${{node%"${{dev}}"}} - "$BUSYBOX" mkdir -p "$("$BUSYBOX" dirname "$name")" - "$BUSYBOX" mknod "$name" "$type" "$major" "$minor" + mkdir -p "$(dirname "$name")" + mknod "$name" "$type" "$major" "$minor" done -"$BUSYBOX" ln -s /proc/self/fd /dev/fd -"$BUSYBOX" ln -s /proc/self/fd/0 /dev/stdin -"$BUSYBOX" ln -s /proc/self/fd/1 /dev/stdout -"$BUSYBOX" ln -s /proc/self/fd/2 /dev/stderr +ln -s /proc/self/fd /dev/fd +ln -s /proc/self/fd/0 /dev/stdin +ln -s /proc/self/fd/1 /dev/stdout +ln -s /proc/self/fd/2 /dev/stderr # Configure networking. -"$BUSYBOX" hostname "$HOSTNAME" -"$BUSYBOX" ip link set lo up +cat << EOF > /etc/hosts +127.0.0.1 localhost +::1 localhost +127.0.1.1 $HOSTNAME.localdomain $HOSTNAME +EOF +: > /etc/resolv.conf +hostname "$HOSTNAME" +ip link set lo up # Find virtio port. vport= for vport_dir in /sys/class/virtio-ports/*; do - if "$BUSYBOX" [ -r "$vport_dir/name" \ - -a "$("$BUSYBOX" cat "$vport_dir/name")" = "$VPORT_NAME" ]; then + if [ -r "$vport_dir/name" -a "$(cat "$vport_dir/name")" = "$VPORT_NAME" ]; then vport="${{vport_dir#/sys/class/virtio-ports/}}" break fi done -if "$BUSYBOX" [ -z "$vport" ]; then - "$BUSYBOX" echo "could not find virtio-port \"$VPORT_NAME\"" +if [ -z "$vport" ]; then + echo "could not find virtio-port \"$VPORT_NAME\"" exit 1 fi cd {cwd} set +e -"$BUSYBOX" sh -c {command} +sh -c {command} rc=$? set -e -"$BUSYBOX" echo "Exited with status $rc" -"$BUSYBOX" echo "$rc" > "/dev/$vport" +echo "Exited with status $rc" +echo "$rc" > "/dev/$vport" """ @@ -208,15 +210,11 @@ def run_in_vm(command: str, kernel_dir: Path, build_dir: Path) -> int: server_sock.bind(str(socket_path)) server_sock.listen() - busybox = shutil.which("busybox") - if busybox is None: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), "busybox") init = (temp_path / "init").resolve() with open(init, "w") as init_file: init_file.write( _INIT_TEMPLATE.format( _9PFS_MSIZE=_9PFS_MSIZE, - busybox=shlex.quote(busybox), python=shlex.quote(sys.executable), cwd=shlex.quote(os.getcwd()), command=shlex.quote(command), @@ -333,7 +331,7 @@ if __name__ == "__main__": kernel_dir = next(download_kernels(args.directory, "x86_64", (kernel,))) try: - command = " ".join(args.command) if args.command else '"$BUSYBOX" sh -i' + command = " ".join(args.command) if args.command else "sh -i" sys.exit(run_in_vm(command, kernel_dir, args.directory)) except LostVMError as e: print("error:", e, file=sys.stderr)