* Merge the boot-order branch. The main improvement is that it uses

mountall to mount filesystems and starts certain jobs (e.g., X11,
  console logins, databases) only after all filesystems have been
  mounted.  If any filesystem fails to mount, an emergency shell is
  started to allow the user to fix the problem.

svn path=/nixos/trunk/; revision=22490

lib/make-iso9660-image.sh

@@ -54,7 +54,7 @@ done
 
 # Also include a manifest of the closures in a format suitable for
 # nix-store --load-db.
-if [ -n "$object"; ]; then
+if [ -n "$object" ]; then
     printRegistration=1 perl $pathsFromGraph closure-* > nix-path-registration
     echo "nix-path-registration=nix-path-registration" >> pathlist
 fi

lib/test-driver/Machine.pm

@@ -2,7 +2,6 @@ package Machine;
 
 use strict;
 use threads;
-use Thread::Queue;
 use Socket;
 use IO::Handle;
 use POSIX qw(dup2);
@@ -28,8 +27,8 @@ sub new {
     if (!$startCommand) {
         # !!! merge with qemu-vm.nix.
         $startCommand =
-            "qemu-system-x86_64 -m 384 -no-kvm-irqchip " .
+            "qemu-system-x86_64 -m 384 " .
-            "-net nic,model=virtio -net user \$QEMU_OPTS ";
+            "-net nic,model=virtio -net user,\$QEMU_NET_OPTS \$QEMU_OPTS ";
         $startCommand .= "-drive file=" . Cwd::abs_path($args->{hda}) . ",if=virtio,boot=on,werror=report "
             if defined $args->{hda};
         $startCommand .= "-cdrom $args->{cdrom} "
@@ -51,7 +50,6 @@ sub new {
         booted => 0,
         pid => 0,
         connected => 0,
-        connectedQueue => Thread::Queue->new(),
         socket => undef,
         stateDir => "$tmpDir/$name",
         monitor => undef,
@@ -101,6 +99,14 @@ sub start {
     bind($monitorS, sockaddr_un($monitorPath)) or die "cannot bind monitor socket: $!";
     listen($monitorS, 1) or die;
 
+    # Create a Unix domain socket to which the root shell in the guest will connect.
+    my $shellPath = $self->{stateDir} . "/shell";
+    unlink $shellPath;
+    my $shellS;
+    socket($shellS, PF_UNIX, SOCK_STREAM, 0) or die;
+    bind($shellS, sockaddr_un($shellPath)) or die "cannot bind shell socket: $!";
+    listen($shellS, 1) or die;
+
     # Start the VM.
     my $pid = fork();
     die if $pid == -1;
@@ -108,22 +114,21 @@ sub start {
     if ($pid == 0) {
         close $serialP;
         close $monitorS;
+        close $shellS;
         open NUL, "</dev/null" or die;
         dup2(fileno(NUL), fileno(STDIN));
         dup2(fileno($serialC), fileno(STDOUT));
         dup2(fileno($serialC), fileno(STDERR));
         $ENV{TMPDIR} = $self->{stateDir};
-        $ENV{QEMU_OPTS} = "-nographic -no-reboot -redir tcp:65535::514 -monitor unix:./monitor";
+        $ENV{USE_TMPDIR} = 1;
+        $ENV{QEMU_OPTS} = "-nographic -no-reboot -monitor unix:./monitor -chardev socket,id=shell,path=./shell";
+        $ENV{QEMU_NET_OPTS} = "guestfwd=tcp:10.0.2.6:23-chardev:shell";
         $ENV{QEMU_KERNEL_PARAMS} = "hostTmpDir=$ENV{TMPDIR}";
         chdir $self->{stateDir} or die;
         exec $self->{startCommand};
         die;
     }
 
-    # Wait until QEMU connects to the monitor.
-    accept($self->{monitor}, $monitorS) or die;
-    $self->waitForMonitorPrompt;
-
     # Process serial line output.
     close $serialC;
 
@@ -131,17 +136,31 @@ sub start {
 
     sub processSerialOutput {
         my ($self, $serialP) = @_;
-        $/ = "\r\n";
         while (<$serialP>) {
             chomp;
+            s/\r$//;
             print STDERR $self->name, "# $_\n";
-            $self->{connectedQueue}->enqueue(1) if $_ eq "===UP===";
         }
-        # If the child dies, wake up connect().
-        $self->{connectedQueue}->enqueue(1);
     }
 
-    $self->log("vm running as pid $pid");
+    eval {
+        local $SIG{CHLD} = sub { die "QEMU died prematurely\n"; };
+        
+        # Wait until QEMU connects to the monitor.
+        accept($self->{monitor}, $monitorS) or die;
+
+        # Wait until QEMU connects to the root shell socket.  QEMU
+        # does so immediately; this doesn't mean that the root shell
+        # has connected yet inside the guest.
+        accept($self->{socket}, $shellS) or die;
+        $self->{socket}->autoflush(1);
+    };
+    die "$@" if $@;
+    
+    $self->waitForMonitorPrompt;
+
+    $self->log("QEMU running (pid $pid)");
+    
     $self->{pid} = $pid;
     $self->{booted} = 1;
 }
@@ -190,27 +209,12 @@ sub connect {
 
     $self->start;
 
-    # Wait until the processQemuOutput thread signals that the machine
+    local $SIG{ALRM} = sub { die "timed out waiting for the guest to connect\n"; };
-    # is up.
+    alarm 300;
-    retry sub {
+    readline $self->{socket} or die;
-        return 1 if $self->{connectedQueue}->dequeue_nb();
+    alarm 0;
-    };
+        
-
+    $self->log("connected to guest root shell");
-    retry sub {
-        $self->log("trying to connect");
-        my $socket = new IO::Handle;
-        $self->{socket} = $socket;
-        socket($socket, PF_UNIX, SOCK_STREAM, 0) or die;
-        connect($socket, sockaddr_un($self->{stateDir} . "/65535.socket")) or die;
-        $socket->autoflush(1);
-        print $socket "echo hello\n" or next;
-        flush $socket;
-        my $line = readline($socket);
-        chomp $line;
-        return 1 if $line eq "hello";
-    };
-
-    $self->log("connected");
     $self->{connected} = 1;
 }
 
@@ -294,7 +298,7 @@ sub waitUntilFails {
 }
 
 
-sub mustFail {
+sub fail {
     my ($self, $command) = @_;
     my ($status, $out) = $self->execute($command);
     die "command `$command' unexpectedly succeeded"
@@ -302,6 +306,11 @@ sub mustFail {
 }
 
 
+sub mustFail {
+    fail @_;
+}
+
+
 # Wait for an Upstart job to reach the "running" state.
 sub waitForJob {
     my ($self, $jobName) = @_;
@@ -360,6 +369,16 @@ sub shutdown {
 }
 
 
+sub crash {
+    my ($self) = @_;
+    return unless $self->{booted};
+
+    $self->sendMonitorCommand("quit");
+
+    $self->waitForShutdown;
+}
+
+
 # Make the machine unreachable by shutting down eth1 (the multicast
 # interface used to talk to the other VMs).  We keep eth0 up so that
 # the test driver can continue to talk to the machine.

lib/test-driver/test-driver.pl

@@ -74,6 +74,3 @@ END {
 
 
 runTests;
-
-
-print STDERR "DONE\n";

modules/config/swap.nix

@@ -0,0 +1,74 @@
+{ config, pkgs, ... }:
+
+with pkgs.lib;
+
+{
+
+  ###### interface
+  
+  options = {
+
+    swapDevices = mkOption {
+      default = [];
+      example = [
+        { device = "/dev/hda7"; }
+        { device = "/var/swapfile"; }
+        { label = "bigswap"; }
+      ];
+      description = ''
+        The swap devices and swap files.  These must have been
+        initialised using <command>mkswap</command>.  Each element
+        should be an attribute set specifying either the path of the
+        swap device or file (<literal>device</literal>) or the label
+        of the swap device (<literal>label</literal>, see
+        <command>mkswap -L</command>).  Using a label is
+        recommended.
+      '';
+
+      type = types.list types.optionSet;
+
+      options = {config, options, ...}: {
+
+        options = {
+        
+          device = mkOption {
+            example = "/dev/sda3";
+            type = types.string;
+            description = "Path of the device.";
+          };
+
+          label = mkOption {
+            example = "swap";
+            type = types.string;
+            description = ''
+              Label of the device.  Can be used instead of <varname>device</varname>.
+            '';
+          };
+
+          cipher = mkOption {
+            default = false;
+            example = true;
+            type = types.bool;
+            description = ''
+              Encrypt the swap device to protect swapped data.  This option
+              does not work with labels.
+            '';
+          };
+
+        };
+
+        config = {
+          device =
+            if options.label.isDefined then
+              "/dev/disk/by-label/${config.label}"
+            else
+              mkNotdef;
+        };
+        
+      };
+      
+    };
+
+  };
+
+}

modules/config/system-path.nix

@@ -22,7 +22,6 @@ let
       pkgs.cpio
       pkgs.curl
       pkgs.diffutils
-      pkgs.e2fsprogs
       pkgs.eject # HAL depends on it anyway
       pkgs.findutils
       pkgs.gawk
@@ -44,7 +43,6 @@ let
       pkgs.pciutils
       pkgs.perl
       pkgs.procps
-      pkgs.reiserfsprogs
       pkgs.rsync
       pkgs.seccure
       pkgs.strace

modules/installer/grub/grub-menu-builder.sh

@@ -216,6 +216,10 @@ extraEntries=`cat <<EOF
 EOF`
 
 
+cat >> $tmp <<EOF
+@extraConfig@
+EOF
+
 if test -n "@extraEntriesBeforeNixOS@"; then 
     echo "$extraEntries" >> $tmp
 fi

modules/installer/grub/grub.nix

@@ -12,7 +12,8 @@ let
     inherit grub;
     inherit (pkgs) bash;
     path = [pkgs.coreutils pkgs.gnused pkgs.gnugrep];
-    inherit (config.boot.loader.grub) copyKernels extraEntries extraEntriesBeforeNixOS
+    inherit (config.boot.loader.grub) copyKernels
+      extraConfig extraEntries extraEntriesBeforeNixOS
       splashImage configurationLimit version default timeout;
   };
   
@@ -67,6 +68,15 @@ in
         '';
       };
 
+      extraConfig = mkOption {
+        default = "";
+        example = "serial; terminal_output.serial";
+        description = ''
+          Additional GRUB commands inserted in the configuration file
+          just before the menu entries.
+        '';
+      };
+
       extraEntries = mkOption {
         default = "";
         example = ''

modules/module-list.nix

@@ -7,6 +7,7 @@
   ./config/no-x-libs.nix
   ./config/nsswitch.nix
   ./config/power-management.nix
+  ./config/swap.nix
   ./config/system-path.nix
   ./config/timezone.nix
   ./config/unix-odbc-drivers.nix
@@ -152,7 +153,6 @@
   ./tasks/kbd.nix
   ./tasks/lvm.nix
   ./tasks/network-interfaces.nix
-  ./tasks/swap.nix
   ./tasks/swraid.nix
   ./tasks/tty-backgrounds.nix
 ]

modules/services/databases/mysql.nix

@@ -100,7 +100,7 @@ in
     jobs.mysql =
       { description = "MySQL server";
 
-        startOn = "started network-interfaces";
+        startOn = "filesystem";
 
         preStart =
           ''

modules/services/databases/postgresql.nix

@@ -28,9 +28,6 @@ let
 
   postgresql = postgresqlAndPlugins pkgs.postgresql;
 
-  startDependency = if config.services.gw6c.enable then 
-    "gw6c" else "network-interfaces";
-
   run = "${pkgs.su}/bin/su -s ${pkgs.stdenv.shell} postgres";
 
   flags = optional cfg.enableTCPIP "-i";
@@ -120,10 +117,10 @@ in
         default = [];
         example = "pkgs.postgis"; # of course don't use a string here!
         description = ''
-          When this list contains elemnts a new store path is created.
+          When this list contains elements a new store path is created.
-          Postgresql and the elments are symlinked into it. Then pg_config,
+          PostgreSQL and the elments are symlinked into it. Then pg_config,
           postgres and pc_ctl are copied to make them use the new
-          $out/lib directory as pkglibdir. This make it possible to use postgis
+          $out/lib directory as pkglibdir. This makes it possible to use postgis
           without patching the .sql files which reference $libdir/postgis-1.5.
         '';
         # Note: the duplication of executables is about 4MB size.
@@ -160,7 +157,7 @@ in
     jobs.postgresql =
       { description = "PostgreSQL server";
 
-        startOn = "started ${startDependency}";
+        startOn = "filesystem";
 
         environment =
           { TZ = config.time.timeZone;

modules/services/hardware/udev.nix

@@ -185,8 +185,14 @@ in
         daemonType = "fork";
 
         exec = "${udev}/sbin/udevd --daemon";
+      };
 
-        postStart =
+    jobs.udevtrigger =
+      { startOn = "started udev";
+
+        task = true;
+
+        script =
           ''
             # Let udev create device nodes for all modules that have already
             # been loaded into the kernel (or for which support is built into
@@ -200,9 +206,8 @@ in
 
             initctl emit -n new-devices
           '';
-        
       };
-
+      
   };
 
 }

modules/services/network-filesystems/nfs-kernel.nix

@@ -8,6 +8,8 @@ let
 
   cfg = config.services.nfsKernel;
 
+  exports = pkgs.writeText "exports" cfg.server.exports;
+
 in
 
 {
@@ -19,7 +21,7 @@ in
     services.nfsKernel = {
 
       client.enable = mkOption {
-        default = false;
+        default = any (fs: fs.fsType == "nfs" || fs.fsType == "nfs4") config.fileSystems;
         description = ''
           Whether to enable the kernel's NFS client daemons.
         '';
@@ -82,7 +84,7 @@ in
       });
 
     environment.etc = mkIf cfg.server.enable (singleton
-      { source = pkgs.writeText "exports" cfg.server.exports;
+      { source = exports;
         target = "exports";
       });
 
@@ -100,19 +102,23 @@ in
               ''
                 export PATH=${pkgs.nfsUtils}/sbin:$PATH
                 mkdir -p /var/lib/nfs
+                
                 ${config.system.sbin.modprobe}/sbin/modprobe nfsd || true
 
+                ${pkgs.sysvtools}/bin/mountpoint -q /proc/fs/nfsd \
+                || ${config.system.sbin.mount}/bin/mount -t nfsd none /proc/fs/nfsd
+
                 ${optionalString cfg.server.createMountPoints
                   ''
                     # create export directories:
                     # skip comments, take first col which may either be a quoted
                     # "foo bar" or just foo (-> man export)
-                    sed '/^#.*/d;s/^"\([^"]*\)".*/\1/;t;s/[ ].*//' ${cfg.server.exports} \
+                    sed '/^#.*/d;s/^"\([^"]*\)".*/\1/;t;s/[ ].*//' ${exports} \
                     | xargs -d '\n' mkdir -p
                   ''
                 }
 
-                # exports file is ${cfg.server.exports}
+                # exports file is ${exports}
                 # keep this comment so that this job is restarted whenever exports changes!
                 exportfs -ra
               '';
@@ -128,7 +134,9 @@ in
             startOn = "started nfs-kernel-exports and started portmap";
             stopOn = "stopping nfs-kernel-exports";
 
-            exec = "${pkgs.nfsUtils}/sbin/rpc.nfsd ${if cfg.server.hostName != null then "-H ${cfg.server.hostName}" else ""} ${builtins.toString cfg.server.nproc}";
+            preStart = "${pkgs.nfsUtils}/sbin/rpc.nfsd ${if cfg.server.hostName != null then "-H ${cfg.server.hostName}" else ""} ${builtins.toString cfg.server.nproc}";
+
+            postStop = "${pkgs.nfsUtils}/sbin/rpc.nfsd 0";
           };
         }
 
@@ -138,10 +146,12 @@ in
 
             description = "Kernel NFS server - mount daemon";
 
-            startOn = "started nfs-kernel-nfsd and started portmap";
+            startOn = "starting nfs-kernel-nfsd and started portmap";
-            stopOn = "stopping nfs-kernel-exports";
+            stopOn = "stopped nfs-kernel-nfsd";
 
-            exec = "${pkgs.nfsUtils}/sbin/rpc.mountd -F -f /etc/exports";
+            daemonType = "fork";
+
+            exec = "${pkgs.nfsUtils}/sbin/rpc.mountd -f /etc/exports";
           };
         }
 
@@ -151,15 +161,34 @@ in
 
             description = "Kernel NFS server - Network Status Monitor";
 
-            startOn = "${if cfg.server.enable then "started nfs-kernel-nfsd and " else ""} started portmap";
+            startOn = "${if cfg.server.enable then "starting nfs-kernel-nfsd and " else ""} started portmap";
-            stopOn = "stopping nfs-kernel-exports";
+            stopOn = "never";
 
             preStart =
               ''	
                 mkdir -p /var/lib/nfs
+                mkdir -p /var/lib/nfs/sm
+                mkdir -p /var/lib/nfs/sm.bak
               '';
 
-            exec = "${pkgs.nfsUtils}/sbin/rpc.statd -F";
+            daemonType = "fork";
+
+            exec = "${pkgs.nfsUtils}/sbin/rpc.statd --no-notify";
+          };
+        }
+      
+      // optionalAttrs (cfg.client.enable || cfg.server.enable)
+        { nfs_kernel_sm_notify = 
+          { name = "nfs-kernel-sm-notify";
+
+            description = "Kernel NFS server - Reboot notification";
+
+            startOn = "started nfs-kernel-statd"
+              + (if cfg.client.enable then " and starting mountall" else "");
+
+            task = true;
+
+            exec = "${pkgs.nfsUtils}/sbin/sm-notify -d";
           };
         };
       

modules/services/networking/portmap.nix

@@ -54,7 +54,7 @@ in
     users.extraUsers = singleton
       { name = "portmap";
         inherit uid;
-        description = "portmap daemon user";
+        description = "Portmap daemon user";
         home = "/var/empty";
       };
 
@@ -66,14 +66,16 @@ in
     jobs.portmap =
       { description = "ONC RPC portmap";
 
-        startOn = "ip-up";
+        startOn = "started network-interfaces";
+        stopOn = "never";
+
+        daemonType = "fork";
 
         exec =
           ''
-            ${portmap}/sbin/portmap -f \
+            ${portmap}/sbin/portmap \
-              ${if config.services.portmap.chroot == ""
+              ${optionalString (config.services.portmap.chroot != "")
-                then ""
+                "-t '${config.services.portmap.chroot}'"} \
-                else "-t \"${config.services.portmap.chroot}\""} \
               ${if config.services.portmap.verbose then "-v" else ""}
           '';
       };

modules/services/ttys/mingetty.nix

@@ -55,7 +55,7 @@ with pkgs.lib;
     # Generate a separate job for each tty.  
     jobs = listToAttrs (map (tty: nameValuePair tty {
     
-      startOn = "started udev";
+      startOn = "started udev and filesystem";
 
       exec = "${pkgs.mingetty}/sbin/mingetty --loginprog=${pkgs.shadow}/bin/login --noclear ${tty}";
       

modules/services/x11/display-managers/kdm.nix

@@ -94,7 +94,9 @@ in
   config = mkIf cfg.enable {
   
     services.xserver.displayManager.job =
-      { execCmd = "PATH=${pkgs.grub}/sbin:$PATH exec ${kdebase_workspace}/bin/kdm -config ${kdmrc} -nodaemon";
+      { execCmd =
+          (optionalString (config.system.boot.loader.id == "grub") "PATH=${config.system.build.grub}/sbin:$PATH ") +
+          "exec ${kdebase_workspace}/bin/kdm -config ${kdmrc} -nodaemon";
         logsXsession = true;
       };
 

modules/services/x11/xserver.nix

@@ -408,7 +408,7 @@ in
       optional (elem "virtualbox" driverNames) kernelPackages.virtualboxGuestAdditions;
 
     jobs.xserver =
-      { startOn = if cfg.autorun then "started udev and started hal" else "";
+      { startOn = if cfg.autorun then "filesystem and stopped udevtrigger and started hal" else "";
  
         environment =
           { FONTCONFIG_FILE = "/etc/fonts/fonts.conf"; # !!! cleanup

modules/system/activation/activation-script.nix

@@ -75,7 +75,6 @@ let
 
     var = fullDepEntry ''
       # Various log/runtime directories.
-      mkdir -m 0755 -p /var/run
 
       touch /var/run/utmp # must exist
       chgrp ${toString config.ids.gids.utmp} /var/run/utmp
@@ -84,16 +83,6 @@ let
       mkdir -m 0755 -p /var/run/nix/current-load # for distributed builds
       mkdir -m 0700 -p /var/run/nix/remote-stores
 
-      # Use a tmpfs for /var/run/nscd to ensure that / or /var can be
-      # unmounted or at least remounted read-only during shutdown.
-      # (Upstart 0.6 apparently uses nscd to do some name lookups,
-      # resulting in it holding some mmap mapping to deleted files in
-      # /var/run/nscd.)
-      if [ ! -e /var/run/nscd ]; then
-          mkdir -p /var/run/nscd
-          ${pkgs.utillinux}/bin/mount -t tmpfs -o "mode=755" none /var/run/nscd
-      fi
-
       mkdir -m 0755 -p /var/log
       mkdir -m 0755 -p /var/log/upstart
 

modules/system/boot/stage-2-init.sh

@@ -109,6 +109,16 @@ rm -rf /var/log/upstart
 rm -rf /nix/var/nix/chroots # recreated in activate-configuration.sh
 
 
+# Use a tmpfs for /var/run to ensure that / or /var can be unmounted
+# or at least remounted read-only during shutdown.  (Upstart 0.6
+# apparently uses nscd to do some name lookups, resulting in it
+# holding some mmap mapping to deleted files in /var/run/nscd.
+# Similarly, portmap and statd have open files in /var/run and are
+# needed during shutdown to unmount NFS volumes.)
+mkdir -m 0755 -p /var/run
+mount -t tmpfs -o "mode=755" none /var/run
+
+
 # Clear the resume device.
 if test -n "$resumeDevice"; then
     mkswap "$resumeDevice" || echo 'Failed to clear saved image.'
@@ -140,9 +150,11 @@ export MODULE_DIR=@kernel@/lib/modules/
 # Run any user-specified commands.
 @shell@ @postBootCommands@
 
+
 # For debugging Upstart.
 #@shell@ --login < /dev/console > /dev/console 2>&1 &
 
+
 # Start Upstart's init.
 echo "starting Upstart..."
 PATH=/var/run/current-system/upstart/sbin exec init

modules/system/upstart-events/runlevel.nix

@@ -4,6 +4,15 @@ with pkgs.lib;
 
 {
 
+  # After booting, go to runlevel 2.  (NixOS doesn't really use
+  # runlevels, but this keeps wtmp happy.)
+  jobs.boot =
+    { name = "boot";
+      startOn = "startup";
+      task = true;
+      script = "telinit 2";
+    };
+
   jobs.runlevel =
     { name = "runlevel";
 

modules/system/upstart-events/shutdown.nix

@@ -37,14 +37,19 @@ with pkgs.lib;
           sync
 
 
-          # Kill all remaining processes except init and this one.
+          # Kill all remaining processes except init, this one and any
+          # Upstart jobs that don't stop on the "starting shutdown"
+          # event, as these are necessary to complete the shutdown.
+          omittedPids=$(initctl list | sed -e 's/.*process \([0-9]\+\)/-o \1/;t;d')
+          #echo "saved PIDs: $omittedPids"
+          
           echo "sending the TERM signal to all processes..."
-          kill -TERM -1
+          ${pkgs.sysvtools}/bin/killall5 -15 $job $omittedPids
       
           sleep 1 # wait briefly
 
           echo "sending the KILL signal to all processes..."
-          kill -KILL -1
+          ${pkgs.sysvtools}/bin/killall5 -9 $job $omittedPids
 
 
           # If maintenance mode is requested, start a root shell, and
@@ -58,38 +63,37 @@ with pkgs.lib;
               initctl emit -n startup
               exit 0
           fi
-                
+
-      
+
+          # Write a shutdown record to wtmp while /var/log is still writable.
+          reboot --wtmp-only
+
+
           # Set the hardware clock to the system time.
           echo "setting the hardware clock..."
           hwclock --systohc --utc
-      
+
-      
+
-          # Unmount helper functions.
+          # Stop all swap devices.
-          getMountPoints() {
+          swapoff -a
-              cat /proc/mounts \
+
-              | grep -v '^rootfs' \
+
-              | sed 's|^[^ ]\+ \+\([^ ]\+\).*|\1|' \
-              | grep -v '/proc\|/sys\|/dev'
-          }
-      
-          getDevice() {
-              local mountPoint=$1
-              cat /proc/mounts \
-              | grep -v '^rootfs' \
-              | grep "^[^ ]\+ \+$mountPoint \+" \
-              | sed 's|^\([^ ]\+\).*|\1|'
-          }
-      
           # Unmount file systems.  We repeat this until no more file systems
           # can be unmounted.  This is to handle loopback devices, file
           # systems  mounted on other file systems and so on.
           tryAgain=1
           while test -n "$tryAgain"; do
               tryAgain=
-      
+              failed= # list of mount points that couldn't be unmounted/remounted
-              for mp in $(getMountPoints); do
+
-                  device=$(getDevice $mp)
+              cp /proc/mounts /dev/.mounts # don't read /proc/mounts while it's changing
+              exec 4< /dev/.mounts
+              while read -u 4 device mp fstype options rest; do
+                  # Skip various special filesystems.  Non-existent
+                  # mount points are typically tmpfs/aufs mounts from
+                  # the initrd.
+                  if [ "$mp" = /proc -o "$mp" = /sys -o "$mp" = /dev -o "$device" = "rootfs" -o "$mp" = /var/run -o ! -e "$mp" ]; then continue; fi
+              
                   echo "unmounting $mp..."
 
                   # We need to remount,ro before attempting any
@@ -99,27 +103,31 @@ with pkgs.lib;
                   # `-i' is to workaround a bug in mount.cifs (it
                   # doesn't recognise the `remount' option, and
                   # instead mounts the FS again).
-                  mount -n -i -o remount,ro "$mp"
+                  success=
+                  if mount -t "$fstype" -n -i -o remount,ro "device" "$mp"; then success=1; fi
 
                   # Note: don't use `umount -f'; it's very buggy.
                   # (For instance, when applied to a bind-mount it
                   # unmounts the target of the bind-mount.)  !!! But
                   # we should use `-f' for NFS.
-                  if umount -n "$mp"; then
+                  if [ "$mp" != / -a "$mp" != /nix/store ]; then
-                      if test "$mp" != /; then tryAgain=1; fi
+                      if umount -n "$mp"; then success=1; tryAgain=1; fi
-                  fi
-      
-                  # Hack: work around a bug in mount (mount -o remount on a
-                  # loop device forgets the loop=/dev/loopN entry in
-                  # /etc/mtab).
-                  if echo "$device" | grep -q '/dev/loop'; then
-                      echo "removing loop device $device..."
-                      losetup -d "$device"
                   fi
+
+                  if [ -z "$success" ]; then failed="$failed $mp"; fi
               done
           done
-      
+
-      
+
+          # Warn about filesystems that could not be unmounted or
+          # remounted read-only.
+          if [ -n "$failed" ]; then
+              echo "warning: the following filesystems could not be unmounted:"
+              for mp in $failed; do echo "  $mp"; done
+              sleep 5
+          fi
+
+
           # Final sync.
           sync
       

modules/tasks/filesystems.nix

@@ -4,103 +4,8 @@ with pkgs.lib;
 
 let
 
-  fileSystems = config.fileSystems;
+  # Packages that provide fsck backends.
-  mount = config.system.sbin.mount;
+  fsPackages = [ pkgs.e2fsprogs pkgs.reiserfsprogs ];
-
-  task =
-    ''
-      PATH=${pkgs.e2fsprogs}/sbin:${pkgs.utillinuxng}/sbin:$PATH
-
-      newDevices=1
-
-      # If we mount any file system, we repeat this loop, because new
-      # mount opportunities may have become available (such as images
-      # for loopback mounts).
-
-      while test -n "$newDevices"; do
-        newDevices=
-
-        ${flip concatMapStrings fileSystems (fs: ''
-          for dummy in x; do # make `continue' work
-            mountPoint='${fs.mountPoint}'
-            device='${if fs.device != null then fs.device else "/dev/disk/by-label/${fs.label}"}'
-            fsType='${fs.fsType}'
-          
-            # A device is a pseudo-device (i.e. not an actual device
-            # node) if it's not an absolute path (e.g. an NFS server
-            # such as machine:/path), if it starts with // (a CIFS FS),
-            # a known pseudo filesystem (such as tmpfs), or the device
-            # is a directory (e.g. a bind mount).
-            isPseudo=
-            test "''${device:0:1}" != / -o "''${device:0:2}" = // -o "$fsType" = "tmpfs" \
-                -o -d "$device" && isPseudo=1
-
-            if ! test -n "$isPseudo" -o -e "$device"; then
-                echo "skipping $device, doesn't exist (yet)"
-                continue
-            fi
-
-            # !!! quick hack: if the mount point is already mounted, try
-            # a remount to change the options but nothing else.
-            if cat /proc/mounts | grep -F -q " $mountPoint "; then
-                if test "''${device:0:2}" != //; then
-                    echo "remounting $device on $mountPoint"
-                    ${mount}/bin/mount -t "$fsType" \
-                        -o remount,"${fs.options}" \
-                        "$device" "$mountPoint" || true
-                fi
-                continue
-            fi
-
-            # If $device is already mounted somewhere else, unmount it first.
-            # !!! Note: we use /etc/mtab, not /proc/mounts, because mtab
-            # contains more accurate info when using loop devices.
-
-            if test -z "$isPseudo"; then
-
-              device=$(readlink -f "$device")
-
-              prevMountPoint=$(
-                  cat /etc/mtab \
-                  | grep "^$device " \
-                  | sed 's|^[^ ]\+ \+\([^ ]\+\).*|\1|' \
-              )
-
-              if test "$prevMountPoint" = "$mountPoint"; then
-                  echo "remounting $device on $mountPoint"
-                  ${mount}/bin/mount -t "$fsType" \
-                      -o remount,"${fs.options}" \
-                      "$device" "$mountPoint" || true
-                  continue
-              fi
-
-              if test -n "$prevMountPoint"; then
-                  echo "unmount $device from $prevMountPoint"
-                  ${mount}/bin/umount "$prevMountPoint" || true
-              fi
-
-            fi
-
-            echo "mounting $device on $mountPoint"
-
-            # !!! should do something with the result; also prevent repeated fscks.
-            if test -z "$isPseudo"; then
-                fsck -a "$device" || true
-            fi
-
-            ${optionalString fs.autocreate
-              ''
-                mkdir -p "$mountPoint"
-              ''
-            }
-
-            if ${mount}/bin/mount -t "$fsType" -o "${fs.options}" "$device" "$mountPoint"; then
-                newDevices=1
-            fi
-          done
-        '')}
-      done
-    '';
 
 in
 
@@ -221,14 +126,99 @@ in
   config = {
 
     # Add the mount helpers to the system path so that `mount' can find them.
-    environment.systemPackages = [pkgs.ntfs3g pkgs.cifs_utils pkgs.nfsUtils];
+    environment.systemPackages =
+      [ pkgs.ntfs3g pkgs.cifs_utils pkgs.nfsUtils pkgs.mountall ]
+      ++ fsPackages;
     
-    jobs.filesystems =
+    environment.etc = singleton
-      { startOn = [ "new-devices" "ip-up" ];
+      { source = pkgs.writeText "fstab"
+          ''
+            # This is a generated file.  Do not edit!
 
-        script = task;
+            # Filesystems.
+            ${flip concatMapStrings config.fileSystems (fs:
+                (if fs.device != null then fs.device else "/dev/disk/by-label/${fs.label}")
+                + " " + fs.mountPoint
+                + " " + fs.fsType
+                + " " + fs.options
+                + " 0"
+                + " " + (if fs.fsType == "none" then "0" else if fs.mountPoint == "/" then "1" else "2")
+                + "\n"
+            )}
+
+            # Swap devices.
+            ${flip concatMapStrings config.swapDevices (sw:
+                 "${sw.device} none swap\n"
+            )}
+          '';
+        target = "fstab";
+      };
+
+    jobs.mountall =
+      { startOn = "started udev";
 
         task = true;
+        
+        script =
+          ''
+            exec > /dev/console 2>&1
+            echo "mounting filesystems..."
+            export PATH=${config.system.sbin.mount}/bin:${makeSearchPath "sbin" ([pkgs.utillinux] ++ fsPackages)}:$PATH
+            ${pkgs.mountall}/sbin/mountall
+          '';
+      };
+
+    # The `mount-failed' event is emitted synchronously, but we don't
+    # want `mountall' to wait for the emergency shell.  So use this
+    # intermediate job to make the event asynchronous.
+    jobs.mountFailed =
+      { name = "mount-failed";
+        task = true;
+        startOn = "mount-failed";
+        script =
+          ''
+            [ -n "$MOUNTPOINT" ] || exit 0
+            start --no-wait emergency-shell \
+              DEVICE="$DEVICE" MOUNTPOINT="$MOUNTPOINT"
+          '';
+      };
+
+    jobs.emergencyShell =
+      { name = "emergency-shell";
+
+        task = true;
+
+        extraConfig = "console owner";
+
+        script =
+          ''
+            [ -n "$MOUNTPOINT" ] || exit 0
+            
+            exec < /dev/console > /dev/console 2>&1
+
+            cat <<EOF
+
+            <<< Emergency shell >>>
+
+            The filesystem \`$DEVICE' could not be mounted on \`$MOUNTPOINT'.
+
+            Please do one of the following:
+
+            - Repair the filesystem (\`fsck $DEVICE') and exit the emergency
+              shell to resume booting.
+
+            - Ignore any failed filesystems and continue booting by running
+              \`initctl emit filesystem'.
+
+            - Remove the failed filesystem from the system configuration in
+              /etc/nixos/configuration.nix and run \`nixos-rebuild switch'.
+            
+            EOF
+
+            ${pkgs.shadow}/bin/login root || false
+
+            initctl start --no-wait mountall
+          '';
       };
 
   };

modules/tasks/swap.nix

@@ -1,133 +0,0 @@
-{ config, pkgs, ... }:
-
-with pkgs.lib;
-
-let
-
-  inherit (pkgs) cryptsetup utillinux;
-
-in
-
-{
-
-  ###### interface
-  
-  options = {
-
-    swapDevices = mkOption {
-      default = [];
-      example = [
-        { device = "/dev/hda7"; }
-        { device = "/var/swapfile"; }
-        { label = "bigswap"; }
-      ];
-      description = ''
-        The swap devices and swap files.  These must have been
-        initialised using <command>mkswap</command>.  Each element
-        should be an attribute set specifying either the path of the
-        swap device or file (<literal>device</literal>) or the label
-        of the swap device (<literal>label</literal>, see
-        <command>mkswap -L</command>).  Using a label is
-        recommended.
-      '';
-
-      type = types.list types.optionSet;
-
-      options = {config, options, ...}: {
-
-        options = {
-          device = mkOption {
-            example = "/dev/sda3";
-            type = types.string;
-            description = ''
-              Path of the device.
-            '';
-          };
-
-          label = mkOption {
-            example = "swap";
-            type = types.string;
-            description = "
-              Label of the device.  Can be used instead of <varname>device</varname>.
-            ";
-          };
-
-          cipher = mkOption {
-            default = false;
-            example = true;
-            type = types.bool;
-            description = "
-              Cipher the swap device to protect swapped data.  This option
-              does not work with labels.
-            ";
-          };
-
-          command = mkOption {
-            description = "
-              Command used to activate the swap device.
-            ";
-          };
-        };
-
-        config = {
-          device =
-            if options.label.isDefined then
-              "/dev/disk/by-label/${config.label}"
-            else
-              mkNotdef;
-
-          command = ''
-            if test -e "${config.device}"; then
-              ${if config.cipher then ''
-                  plainDevice="${config.device}"
-                  name="crypt$(echo "$plainDevice" | sed -e 's,/,.,g')"
-                  device="/dev/mapper/$name"
-                  if ! test -e "$device"; then
-                    ${cryptsetup}/sbin/cryptsetup -c aes -s 128 -d /dev/urandom create "$name" "$plainDevice"
-                    ${utillinux}/sbin/mkswap -f "$device" || true
-                  fi
-                ''
-                else ''
-                  device="${config.device}"
-                ''
-              }
-              device=$(readlink -f "$device")
-              # Add new swap devices.
-              if echo $unused | grep -q "^$device\$"; then
-                unused="$(echo $unused | grep -v "^$device\$" || true)"
-              else
-                ${utillinux}/sbin/swapon "$device" || true
-              fi
-            fi
-          '';
-        };
-
-      };
-      
-    };
-
-  };
-
-  ###### implementation
-
-  config = {
-
-    jobs.swap =
-      { task = true;
-
-        startOn = ["startup" "new-devices"];
-
-        script =
-          ''
-            unused="$(sed '1d; s/ .*//' /proc/swaps)"
-
-            ${toString (map (x: x.command) config.swapDevices)}
-
-            # Remove remaining swap devices.
-            test -n "$unused" && ${utillinux}/sbin/swapoff $unused || true
-          '';
-      };
-
-  };
-
-}

modules/testing/test-instrumentation.nix

@@ -13,6 +13,7 @@ let
     ''
       #! ${pkgs.perl}/bin/perl
       $SIG{CHLD} = 'DEFAULT';
+      print "\n";
       exec "/bin/sh";
     '';
 
@@ -23,14 +24,9 @@ in
   config = {
 
     jobs.backdoor =
-      { startOn = "started network-interfaces";
+      { startOn = "ip-up";
+        stopOn = "never";
         
-        preStart =
-          ''
-            echo "guest running" > /dev/ttyS0
-            echo "===UP===" > dev/ttyS0
-          '';
-          
         script =
           ''
             export USER=root
@@ -39,10 +35,13 @@ in
             export GCOV_PREFIX=/tmp/coverage-data
             source /etc/profile
             cd /tmp
-            exec ${pkgs.socat}/bin/socat tcp-listen:514,fork exec:${rootShell} 2> /dev/ttyS0
+            echo "connecting to host..." > /dev/ttyS0
+            ${pkgs.socat}/bin/socat tcp:10.0.2.6:23 exec:${rootShell} 2> /dev/ttyS0 || poweroff -f
           '';
+
+        respawn = false;
       };
-  
+
     boot.postBootCommands =
       ''
         # Panic on out-of-memory conditions rather than letting the
@@ -75,10 +74,19 @@ in
     # `xwininfo' is used by the test driver to query open windows.
     environment.systemPackages = [ pkgs.xorg.xwininfo ];
 
-    # Send all of /var/log/messages to the serial port (except for
+    # Send all of /var/log/messages to the serial port.
-    # kernel messages through klogd, which already appear on the
+    services.syslogd.extraConfig = "*.* /dev/ttyS0";
-    # serial port).
+
-    services.syslogd.extraConfig = "*.*,kern.none /dev/ttyS0";
+    # Prevent tests from accessing the Internet.
+    networking.defaultGateway = mkOverride 200 {} "";
+    networking.nameservers = mkOverride 200 {} [ ];
+
+    # Apply a patch to the kernel to increase the 15s CIFS timeout.
+    nixpkgs.config.packageOverrides = pkgs: {
+      linux = pkgs.linux.override (orig: {
+        kernelPatches = orig.kernelPatches ++ [ pkgs.kernelPatches.cifs_timeout ];
+      });
+    };
 
   };
 

modules/virtualisation/qemu-vm.nix

@@ -113,20 +113,29 @@ let
     ''
       #! ${pkgs.stdenv.shell}
       
-      export PATH=${pkgs.samba}/sbin:$PATH
+      NIX_DISK_IMAGE=$(readlink -f ''${NIX_DISK_IMAGE:-${config.virtualisation.diskImage}})
-
-      NIX_DISK_IMAGE=''${NIX_DISK_IMAGE:-${config.virtualisation.diskImage}}
 
       if ! test -e "$NIX_DISK_IMAGE"; then
-          ${pkgs.qemu_kvm}/bin/qemu-img create -f qcow2 "$NIX_DISK_IMAGE" ${toString config.virtualisation.diskSize}M || exit 1
+          ${pkgs.qemu_kvm}/bin/qemu-img create -f qcow2 "$NIX_DISK_IMAGE" \
+            ${toString config.virtualisation.diskSize}M || exit 1
       fi
-      
+
-      # -no-kvm-irqchip is needed to prevent the CIFS mount from
+      # Start Samba (which wants to put its socket and config files in TMPDIR).
-      # hanging the VM on x86_64.
+      if [ -z "$TMPDIR" -o -z "$USE_TMPDIR" ]; then
-      exec ${pkgs.qemu_kvm}/bin/qemu-system-x86_64 -m ${toString config.virtualisation.memorySize} \
+          TMPDIR=$(mktemp -d nix-vm-smbd.XXXXXXXXXX --tmpdir)
-          -no-kvm-irqchip \
+      fi
-          -net nic,vlan=0,model=virtio -net user,vlan=0 -smb / \
+      cd $TMPDIR
-          -drive file=$NIX_DISK_IMAGE,if=virtio,boot=on,werror=report \
+
+      ${pkgs.vmTools.startSamba}
+
+      # Start QEMU.
+      exec ${pkgs.qemu_kvm}/bin/qemu-system-x86_64 \
+          -name ${vmName} \
+          -m ${toString config.virtualisation.memorySize} \
+          -net nic,vlan=0,model=virtio \
+          -chardev socket,id=samba,path=./samba \
+          -net user,vlan=0,guestfwd=tcp:10.0.2.4:139-chardev:samba''${QEMU_NET_OPTS:+,$QEMU_NET_OPTS} \
+          -drive file=$NIX_DISK_IMAGE,if=virtio,boot=on,cache=writeback,werror=report \
           -kernel ${config.system.build.toplevel}/kernel \
           -initrd ${config.system.build.toplevel}/initrd \
           ${qemuGraphics} \
@@ -167,6 +176,9 @@ in
       
   boot.initrd.postDeviceCommands =
     ''
+      # Workaround for massive clock drift with the "kvm-clock" clock source.
+      echo hpet > /sys/devices/system/clocksource/clocksource0/current_clocksource
+    
       # Set up networking.  Needed for CIFS mounting.
       ifconfig eth0 up 10.0.2.15
 
@@ -212,6 +224,7 @@ in
       }
       { mountPoint = "/nix/store";
         device = "/hostfs/nix/store";
+        fsType = "none";
         options = "bind";
         neededForBoot = true;
       }

release.nix

@@ -111,10 +111,10 @@ let
         options =
           (import lib/eval-config.nix {
             inherit nixpkgs;
-            modules = [ ];
+            modules = [ { fileSystems = []; } ];
           }).options;
-        revision = with nixosSrc;
+        revision = 
-          if rev == 1234 then "HEAD" else toString rev;
+          if nixosSrc.rev == 1234 then "HEAD" else toString nixosSrc.rev;
       };
 
 
@@ -156,6 +156,8 @@ let
         installer.swraid = t.installer.swraid.test;
         kde4 = t.kde4.test;
         login = t.login.test;
+        nat = t.nat.test;
+        nfs = t.nfs.test;
         openssh = t.openssh.test;
         proxy = t.proxy.test;
         quake3 = t.quake3.test;

tests/default.nix

@@ -12,6 +12,7 @@ with import ../lib/testing.nix { inherit nixpkgs services system; };
   kde4 = makeTest (import ./kde4.nix);
   login = makeTest (import ./login.nix);
   nat = makeTest (import ./nat.nix);
+  nfs = makeTest (import ./nfs.nix);
   openssh = makeTest (import ./openssh.nix);
   portmap = makeTest (import ./portmap.nix);
   proxy = makeTest (import ./proxy.nix);

tests/installer.nix

@@ -40,6 +40,7 @@ let
 
         boot.loader.grub.version = 2;
         boot.loader.grub.device = "/dev/vda";
+        boot.loader.grub.extraConfig = "serial; terminal_output.serial";
         boot.initrd.kernelModules = [ "ext3" ];
       
         fileSystems = [ ${fileSystems} ];
@@ -218,9 +219,7 @@ in {
         ''
           $machine->mustSucceed(
               "parted /dev/vda mklabel msdos",
-              "udevadm settle",
               "parted /dev/vda -- mkpart primary linux-swap 1M 1024M",
-              "udevadm settle",
               "parted /dev/vda -- mkpart primary ext2 1024M -1s",
               "udevadm settle",
               "mkswap /dev/vda1 -L swap",
@@ -240,9 +239,7 @@ in {
           $machine->mustSucceed(
               "parted /dev/vda mklabel msdos",
               "parted /dev/vda -- mkpart primary ext2 1M 50MB", # /boot
-              "udevadm settle",
               "parted /dev/vda -- mkpart primary linux-swap 50MB 1024M",
-              "udevadm settle",
               "parted /dev/vda -- mkpart primary ext2 1024M -1s", # /
               "udevadm settle",
               "mkswap /dev/vda2 -L swap",
@@ -265,11 +262,8 @@ in {
           $machine->mustSucceed(
               "parted /dev/vda mklabel msdos",
               "parted /dev/vda -- mkpart primary 1M 2048M", # first PV
-              "udevadm settle",
               "parted /dev/vda -- set 1 lvm on",
-              "udevadm settle",
               "parted /dev/vda -- mkpart primary 2048M -1s", # second PV
-              "udevadm settle",
               "parted /dev/vda -- set 2 lvm on",
               "udevadm settle",
               "pvcreate /dev/vda1 /dev/vda2",
@@ -290,21 +284,25 @@ in {
         ''
           $machine->mustSucceed(
               "parted /dev/vda mklabel msdos",
-              "parted /dev/vda -- mkpart primary 1M 1000M", # md0 (root), first device
+              "parted /dev/vda -- mkpart primary ext2 1M 30MB", # /boot
-              "parted /dev/vda -- mkpart primary 1024M 2024M", # md0 (root), second device
+              "parted /dev/vda -- mkpart extended 30M -1s", # extended partition
-              "parted /dev/vda -- mkpart primary 2048M 2548M", # md1 (swap), first device
+              "parted /dev/vda -- mkpart logical 30M 1000M", # md0 (root), first device
-              "parted /dev/vda -- mkpart primary 2560M 3060M", # md1 (swap), second device
+              "parted /dev/vda -- mkpart logical 1024M 2000M", # md0 (root), second device
+              "parted /dev/vda -- mkpart logical 2048M 2548M", # md1 (swap), first device
+              "parted /dev/vda -- mkpart logical 2560M 3060M", # md1 (swap), second device
               "udevadm settle",
-              # Note that GRUB2 doesn't work with version 1.2 metadata.
+              "mdadm --create --force /dev/md0 --metadata 1.2 --level=raid1 --raid-devices=2 /dev/vda5 /dev/vda6",
-              "mdadm --create --force /dev/md0 --metadata 0.90 --level=raid1 --raid-devices=2 /dev/vda1 /dev/vda2",
+              "mdadm --create --force /dev/md1 --metadata 1.2 --level=raid1 --raid-devices=2 /dev/vda7 /dev/vda8",
-              "mdadm --create --force /dev/md1 --metadata 1.2 --level=raid1 --raid-devices=2 /dev/vda3 /dev/vda4",
               "mkswap -f /dev/md1 -L swap",
               "swapon -L swap",
               "mkfs.ext3 -L nixos /dev/md0",
               "mount LABEL=nixos /mnt",
+              "mkfs.ext3 -L boot /dev/vda1",
+              "mkdir /mnt/boot",
+              "mount LABEL=boot /mnt/boot",
           );
         '';
-      fileSystems = rootFS;
+      fileSystems = rootFS + bootFS;
     };
       
 }

tests/nfs.nix

@@ -0,0 +1,84 @@
+{ pkgs, ... }:
+
+let
+
+  client = 
+    { config, pkgs, ... }:
+    { fileSystems = pkgs.lib.mkOverride 50 {} 
+        [ { mountPoint = "/data";
+            device = "server:/data";
+            fsType = "nfs";
+            options = "bootwait";
+          } 
+        ];
+    };
+
+in
+
+{
+
+  nodes =
+    { client1 = client;
+      client2 = client;
+
+      server = 
+        { config, pkgs, ... }:
+        { services.nfsKernel.server.enable = true;
+          services.nfsKernel.server.exports =
+            ''
+              /data 192.168.1.0/255.255.255.0(rw,no_root_squash)
+            '';
+          services.nfsKernel.server.createMountPoints = true;
+        };
+    };
+
+  testScript =
+    ''
+      startAll;
+
+      $server->waitForJob("nfs-kernel-nfsd");
+      $server->waitForJob("nfs-kernel-mountd");
+      $server->waitForJob("nfs-kernel-statd");
+
+      $client1->waitForJob("tty1"); # depends on filesystems
+      $client1->succeed("echo bla > /data/foo");
+      $server->succeed("test -e /data/foo");
+
+      $client2->waitForJob("tty1"); # depends on filesystems
+      $client2->succeed("echo bla > /data/bar");
+      $server->succeed("test -e /data/bar");
+
+      # Test whether we can get a lock.  !!! This step takes about 90
+      # seconds because the NFS server waits that long after booting
+      # before accepting new locks.
+      $client2->succeed("time flock -n -s /data/lock true");
+      
+      # Test locking: client 1 acquires an exclusive lock, so client 2
+      # should then fail to acquire a shared lock.
+      $client1->succeed("flock -x /data/lock -c 'touch locked; sleep 100000' &");
+      $client1->waitForFile("locked");
+      $client2->fail("flock -n -s /data/lock true");
+
+      # Test whether client 2 obtains the lock if we reset client 1.
+      $client2->succeed("flock -x /data/lock -c 'echo acquired; touch locked; sleep 100000' >&2 &");
+      $client1->crash;
+      $client1->start;
+      $client2->waitForFile("locked");
+
+      # Test whether locks survive a reboot of the server.
+      $client1->waitForJob("tty1"); # depends on filesystems
+      $server->shutdown;
+      $server->start;
+      $client1->succeed("touch /data/xyzzy");
+      $client1->fail("time flock -n -s /data/lock true");
+
+      # Test whether unmounting during shutdown happens quickly.  This
+      # requires portmap and statd to keep running during the
+      # shutdown.
+      my $t1 = time;
+      $client1->shutdown;
+      my $duration = time - $t1;
+      die "shutdown took too long ($duration seconds)" if $duration > 30;
+    '';
+
+}