* Cleanup.

svn path=/nixpkgs/trunk/; revision=11166

pkgs/build-support/vm/rpm/fedora-postinstall.sh

@@ -1,5 +0,0 @@
-# Add missing symlink to /lib/libssl.so.0.9.7a (or similar).  Curl
-# needs it.
-echo "adding libssl symlink..."
-chroot $root /bin/sh -c "/bin/ln -v -s /lib/libssl.so.* /lib/libssl.so.4"
-chroot $root /bin/sh -c "/bin/ln -v -s /lib/libcrypto.so.* /lib/libcrypto.so.4"

pkgs/build-support/vm/rpm/fill-disk-with-rpms.sh

@@ -1,43 +0,0 @@
-source $stdenv/setup
-
-size=$(($size * 1024 * 1024))
-
-image=$out/image
-ensureDir $out
-
-echo "building image \`$fullName'"
-echo "$name" > $out/name
-echo "$fullName" > $out/full-name
-
-echo "creating sparse file of $size bytes in $image..."
-dd if=/dev/zero of=$image bs=1 seek=$((size - 1)) count=1
-
-echo "creating e2fs file system in $image..."
-$e2fsprogs/sbin/mke2fs -F $image
-
-echo "initialising file system through UML..."
-echo $PATH > path
-echo $rpm > rpm
-echo $sysvinit > sysvinit
-echo $rpms > rpms
-echo $postInstall > post-install
-ln -s $worker $NIX_BUILD_TOP/worker # work around a stupid bug (no dots allowed in kernel arguments?)
-
-# UML requires an existing $HOME.
-mkdir dummy
-export HOME=$(pwd)/dummy
-
-# Run UML.
-touch log
-tail -n +0 -f log & # show UML output as it appears
-tailPid=$!
-
-linux ubd0=$image root=/dev/root rootflags=/ rootfstype=hostfs \
-  init="$SHELL $NIX_BUILD_TOP/worker $NIX_BUILD_TOP" con=null || true
-  
-echo image is $image
-
-sleep 1 # drain `tail', hacky
-kill $!
-
-if ! test -e success; then exit 1; fi

pkgs/build-support/vm/rpm/fill-disk-worker.sh

@@ -1,65 +0,0 @@
-set -e
-set -x
-
-export NIX_BUILD_TOP=$1
-
-export PATH=$(< $NIX_BUILD_TOP/path)
-export sysvinit=$(< $NIX_BUILD_TOP/sysvinit)
-export postInstall=$(< $NIX_BUILD_TOP/post-install)
-export rpm=$(< $NIX_BUILD_TOP/rpm)
-
-cd $NIX_BUILD_TOP
-
-root=$NIX_BUILD_TOP/root
-
-echo "remounting / writeable..."
-mount -n -o remount,rw dummy /
-
-exec > log 2>&1
-
-echo "creating /dev..."
-mount -n -t tmpfs none /dev
-mknod /dev/ubd0 b 98 0
-
-echo "mounting image on $root..."
-mkdir root
-mount -n -t ext2 /dev/ubd0 $root
-
-echo "making directories..."
-mkdir --mode 0755 $root/dev
-mkdir --mode 0755 $root/proc
-
-echo "initialising RPM DB..."
-rpm="$rpm/bin/rpm --root $root --dbpath /var/lib/rpm"
-$rpm --initdb
-
-echo "installing RPMs..."
-$rpm --noscripts --notriggers --nodeps -ihv $(cat rpms)
-
-if test -n "$postInstall"; then
-    echo "running post-install script"
-    . $postInstall
-fi
-
-# Get rid of the Berkeley DB environment so that older RPM versions
-# (using older versions of BDB) will still work.
-rm -f root/var/lib/rpm/__db.*
-
-# We need to make devices because modern distributions (e.g., Fedora
-# Core 3) use udev to create devices dynamically.  We run into *very*
-# subtle bugs if /dev/null in particular doesn't exist ;-)
-echo "making devices..."
-test -e $root/dev/null || mknod -m 0666 $root/dev/null c 1 3
-test -e $root/dev/zero || mknod -m 0666 $root/dev/zero c 1 5
-test -e $root/dev/tty  || mknod -m 0600 $root/dev/tty  c 5 0
-
-echo "unmounting..."
-umount -n $root
-
-touch $NIX_BUILD_TOP/success
-    
-echo "syncing..."
-sync
-
-echo "halting..."
-$sysvinit/sbin/halt -d -f

pkgs/build-support/vm/rpm/redhat-postinstall.sh

@@ -1,4 +0,0 @@
-# This is needed by curl.
-echo "adding /usr/kerberos/lib to ld.so.conf..."
-echo "/usr/kerberos/lib" >> $root/etc/ld.so.conf
-chroot $root /sbin/ldconfig

pkgs/build-support/vm/rpm/run-in-uml-boot.sh

@@ -1,59 +0,0 @@
-set -e
-set -x
-
-# Restore the original variables.
-export NIX_BUILD_TOP=$1
-. $NIX_BUILD_TOP/initial-env-vars
-
-export PATH=$(< $NIX_BUILD_TOP/path)
-
-export NIX_STORE=/nix/store # !!! assuming that Nix store = /nix/store
-
-cd $NIX_BUILD_TOP
-
-root=$NIX_BUILD_TOP/root
-
-echo "remounting / writeable..."
-mount -n -o remount,rw dummy /
-
-exec > log 2>&1
-
-echo "creating /dev..."
-mount -n -t tmpfs none /dev
-mknod /dev/ubd0 b 98 0
-
-echo "mounting image on $root..."
-mkdir root
-mount -n -t ext2 /dev/ubd0 $root
-
-echo "switching roots"
-mkdir $root/hostfs
-
-# Make the Nix store available in the UML file system as a symlink to /hostfs/$NIX_STORE.
-mkdir -p $root/$(dirname $NIX_STORE)
-ln -s /hostfs/$NIX_STORE $root/$NIX_STORE 
-
-$utillinux/sbin/pivot_root $root $root/hostfs
-cd /
-# !!! we should also chroot here (according to the pivot_root manpage)
-
-echo "mounting /proc..."
-mount -n -t proc none /proc
-
-#mount --bind /hostfs/tmp /tmp
-
-cd /tmp
-export TMPDIR=/tmp
-export TMP_DIR=/tmp
-#cd $NIX_BUILD_TOP
-
-# Call the actual builder.
-if sh -e $actualBuilder; then
-    touch /hostfs/$NIX_BUILD_TOP/success
-fi
-
-echo "syncing..."
-sync
-
-echo "halting..."
-$sysvinit/sbin/halt -d -f

pkgs/build-support/vm/rpm/run-in-uml.sh

@@ -1,35 +0,0 @@
-export > initial-env-vars
-
-source $stdenv/setup
-
-ensureDir $out/nix-support
-echo "$system" > $out/nix-support/system
-cat $image/name > $out/nix-support/name
-cat $image/full-name > $out/nix-support/full-name
-
-echo $PATH > path
-ln -s $boot $NIX_BUILD_TOP/worker # work around a stupid bug (no dots allowed in kernel arguments?)
-
-# UML requires an existing $HOME.
-mkdir dummy
-export HOME=$(pwd)/dummy
-
-# Run UML.
-startLog "uml-run"
-touch log
-tail -n +0 -f log & # show UML output as it appears
-tailPid=$!
-
-echo "running UML using image $image/image ($(cat $image/full-name))"
-linux mem=384M ubd0=cow,$image/image root=/dev/root rootflags=/ rootfstype=hostfs \
-  init="$SHELL $NIX_BUILD_TOP/worker $NIX_BUILD_TOP" con=null || true
-echo "UML finished"
-
-sleep 1 # drain `tail', hacky
-kill $!
-stopLog
-
-if ! test -e success; then
-    echo "UML build script failed"
-    exit 1
-fi

pkgs/build-support/vm/test.nix

@@ -0,0 +1,57 @@
+with import ../../.. {};
+with import ./vm.nix;
+
+rec {
+
+
+  # Run the PatchELF derivation in a VM.
+  buildPatchelfInVM = runInLinuxVM patchelf;
+
+
+  rpmImage = fillDiskWithRPMs {
+    name = "fedora-image";
+    fullName = "Fedora Core 3";
+    size = 1024;
+    rpms = import ./rpm/fedora-3-packages.nix {inherit fetchurl;};
+  };
+
+
+  testRPMImage = makeImageTestScript rpmImage;
+
+
+  buildPatchelfRPM = buildRPM {
+    name = "patchelf-rpm";
+    src = patchelf.src;
+    diskImage = rpmImage;
+  };
+
+  
+  ubuntuImage = fillDiskWithDebs {
+    name = "ubuntu-image";
+    fullName = "Ubuntu 7.10 Gutsy";
+    size = 256;
+    debs = import ./deb/ubuntu-7.10-gutsy-i386.nix {inherit fetchurl;};
+  };
+  
+
+  debianImage = fillDiskWithDebs {
+    name = "debian-image";
+    fullName = "Debian 4.0r3 Etch";
+    size = 256;
+    debs = import ./deb/debian-4.0r3-etch-i386.nix {inherit fetchurl;};
+  };
+
+
+  testUbuntuImage = makeImageTestScript ubuntuImage;
+
+  
+  buildInDebian = runInLinuxImage (stdenv.mkDerivation {
+    name = "deb-compile";
+    src = nixUnstable.src;
+    diskImage = debianImage;
+    memSize = 512;
+  });
+  
+
+  
+}

pkgs/build-support/vm/vm.nix

@@ -176,7 +176,7 @@ rec {
 
     PATH=${coreutils}/bin:${kvm}/bin
 
-    diskImage=/dev/null
+    diskImage=''${diskImage:-/dev/null}
 
     eval "$preVM"
 
@@ -201,8 +201,52 @@ rec {
     exit $(cat in-vm-exit)
   '';
 
-  
+
-  # Modify the given derivation to perform it in a virtual machine.
+  createEmptyImage = {size, fullName}: ''
+    mkdir $out
+    diskImage=$out/image
+    qemu-img create -f qcow $diskImage "${toString size}M"
+
+    mkdir $out/nix-support
+    echo ${fullName} > $out/nix-support/full-name
+  '';
+
+
+  createRootFS = ''
+    mkdir /mnt
+    ${e2fsprogs}/sbin/mke2fs -F /dev/sda
+    ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt
+
+    if test -e /mnt/.debug; then
+      exec ${bash}/bin/sh
+    fi
+    touch /mnt/.debug
+
+    mkdir /mnt/proc /mnt/dev /mnt/sys /mnt/bin
+  '';
+
+
+  /* Run a derivation in a Linux virtual machine (using Qemu/KVM).  By
+     default, there is no disk image; the root filesystem is a tmpfs,
+     and /nix/store is shared with the host (via the CIFS protocol to
+     a Samba instance automatically started by Qemu).  Thus, any pure
+     Nix derivation should run unmodified, e.g. the call
+
+       runInLinuxVM patchelf
+
+     will build the derivation `patchelf' inside a VM.  The attribute
+     `preVM' can optionally contain a shell command to be evaluated
+     *before* the VM is started (i.e., on the host).  The attribute
+     `memSize' specifies the memory size of the VM in megabytes,
+     defaulting to 256.  The attribute `diskImage' can optionally
+     specify a file system image to be attached to /dev/sda.  (Note
+     that currently we expect the image to contain a filesystem, not a
+     full disk image with a partition table etc.)
+
+     If the build fails and Nix is run with the `-K' option, a script
+     `run-vm' will be left behind in the temporary build directory
+     that allows you to boot into the VM and debug it interactively. */
+     
   runInLinuxVM = attrs: derivation (removeAttrs attrs ["meta" "passthru" "outPath" "drvPath"] // {
     builder = "${bash}/bin/sh";
     args = ["-e" vmRunCommand];
@@ -211,28 +255,50 @@ rec {
     QEMU_OPTS = "-m ${toString (if attrs ? memSize then attrs.memSize else 256)}";
   });
 
-  
+
-  test = runInLinuxVM patchelf;
+  /* Like runInLinuxVM, but run the build not using the stdenv from
+     the Nix store, but using the tools provided by /bin, /usr/bin
+     etc. from the specified filesystem image, which typically is a
+     filesystem containing a non-NixOS Linux distribution. */
+     
+  runInLinuxImage = attrs: runInLinuxVM (attrs // {
+    mountDisk = true;
+
+    /* Mount `image' as the root FS, but use a temporary copy-on-write
+       image since we don't want to (and can't) write to `image'. */
+    preVM = ''
+      diskImage=$(pwd)/image
+      origImage=${attrs.diskImage}
+      if test -d "$origImage"; then origImage="$origImage/image"; fi
+      qemu-img create -b "$origImage" -f qcow $diskImage
+    '';
+
+    /* Inside the VM, run the stdenv setup script normally, but at the
+       very end set $PATH and $SHELL to the `native' paths for the
+       distribution inside the VM. */
+    postHook = ''
+      PATH=/usr/bin:/bin:/usr/sbin:/sbin
+      SHELL=/bin/sh
+    '';
+
+    /* Don't run Nix-specific build steps like patchelf. */
+    fixupPhase = "true";
+  });
 
 
+  /* Create a filesystem image of the specified size and fill it with
+     a set of RPM packages. */
+    
   fillDiskWithRPMs =
     {size ? 1024, rpms, name, fullName, postInstall ? null}:
     
     runInLinuxVM (stdenv.mkDerivation {
       inherit name postInstall rpms;
 
-      preVM = ''
+      preVM = createEmptyImage {inherit size fullName;};
-        mkdir $out
-        diskImage=$out/image
-        qemu-img create -f qcow $diskImage "${toString size}M"
-      '';
 
       buildCommand = ''
-        mkdir /mnt
+        ${createRootFS}
-        ${e2fsprogs}/sbin/mke2fs -F /dev/sda
-        ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt
-
-        mkdir /mnt/proc /mnt/dev /mnt/sys
 
         echo "initialising RPM DB..."
         rpm="${rpm}/bin/rpm --root /mnt --dbpath /var/lib/rpm"
@@ -254,21 +320,16 @@ rec {
         echo "running post-install script..."
         eval "$postInstall"
         
+        rm /mnt/.debug
+        
         ${klibcShrunk}/bin/umount /mnt
       '';
     });
 
 
-  test2 = fillDiskWithRPMs {
+  /* Generate a script that can be used to run an interactive session
-    size = 1024;
+     in the given image. */
-    name = "testY";
+     
-    fullName = "Test Image";
-    rpms = import ./rpm/fedora-3-packages.nix {inherit fetchurl;};
-  };
-
-
-  # Generates a script that can be used to run an interactive session
-  # in the given image.
   makeImageTestScript = image: writeScript "image-test" ''
     #! ${bash}/bin/sh
     if test -z "$1"; then
@@ -287,39 +348,36 @@ rec {
     ${qemuCommand}
   '';
 
-  test3 = makeImageTestScript test2;
 
-
+  /* Build RPM packages from the tarball `src' in the Linux
-  buildRPM = runInLinuxVM (stdenv.mkDerivation {
+     distribution installed in the filesystem `diskImage'.  The
-    name = "rpm-test";
+     tarball must contain an RPM specfile. */
   
-    mountDisk = true;
+  buildRPM = attrs: runInLinuxImage (stdenv.mkDerivation (attrs // {
+    phases = "buildPhase installPhase";
   
-    preVM = ''
+    buildPhase = ''
-      diskImage=$(pwd)/image
+      # Hacky: RPM looks for <basename>.spec inside the tarball, so
-      qemu-img create -b ${test2}/image -f qcow $diskImage
+      # strip off the hash.
-    '';
-
-    src = patchelf.src;
-
-    buildCommand = ''
-      PATH=/usr/bin:/bin:/usr/sbin:/sbin
-
-      echo ${patchelf.src}
-
       stripHash "$src"
       srcName="$strippedName"
       ln -s "$src" "$srcName"
 
       rpmbuild -vv -ta "$srcName"
+    '';
 
+    installPhase = ''
       ensureDir $out/rpms
       find /usr/src -name "*.rpm" -exec cp {} $out/rpms \;
     '';
-  });
+  }));
 
 
-  # !!! should probably merge this with fillDiskWithRPMs.
+  /* Create a filesystem image of the specified size and fill it with
+     a set of Debian packages.  `debs' must be a list of list of
+     .deb files, namely, the Debian packages grouped together into
+     strongly connected components.  See deb/deb-closure.nix. */
+
   fillDiskWithDebs =
     {size ? 1024, debs, name, fullName, postInstall ? null}:
     
@@ -328,23 +386,10 @@ rec {
 
       debs = (lib.intersperse "|" debs);
 
-      preVM = ''
+      preVM = createEmptyImage {inherit size fullName;};
-        mkdir $out
-        diskImage=$out/image
-        qemu-img create -f qcow $diskImage "${toString size}M"
-      '';
 
       buildCommand = ''
-        mkdir /mnt
+        ${createRootFS}
-        ${e2fsprogs}/sbin/mke2fs -F /dev/sda
-        ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt
-
-        if test -e /mnt/.debug; then
-          exec ${bash}/bin/sh
-        fi
-        touch /mnt/.debug
-
-        mkdir /mnt/proc /mnt/dev /mnt/sys /mnt/bin
 
         echo "initialising Debian DB..."
         PATH=$PATH:${dpkg}/bin:${dpkg}/sbin:${glibc}/sbin
@@ -406,45 +451,4 @@ rec {
     });
 
 
-  test4 = fillDiskWithDebs {
-    size = 256;
-    name = "deb-test";
-    fullName = "Ubuntu Test Image";
-    debs = import ./deb/ubuntu-7.10-gutsy-i386.nix {inherit fetchurl;};
-  };
-
-  test5 = makeImageTestScript test4;
-
-  
-  test6 = runInLinuxVM (stdenv.mkDerivation {
-    name = "deb-compile";
-
-    mountDisk = true;
-  
-    preVM = ''
-      diskImage=$(pwd)/image
-      qemu-img create -b ${test7}/image -f qcow $diskImage
-    '';
-
-    src = nixUnstable.src;
-
-    postHook = ''
-      PATH=/usr/bin:/bin:/usr/sbin:/sbin
-      SHELL=/bin/sh
-    '';
-
-    fixupPhase = "true";
-
-    memSize = 512;
-  });
-  
-
-  test7 = fillDiskWithDebs {
-    size = 256;
-    name = "deb-test";
-    fullName = "Debian Test Image";
-    debs = import ./deb/debian-4.0r3-etch-i386.nix {inherit fetchurl;};
-  };
-
-
 }