Make it easier to run the tests

You can now run a test in the nixos/tests directory directly using
nix-build, e.g.

  $ nix-build '<nixos/tests/login.nix>' -A test

This gets rid of having to add the test to nixos/tests/default.nix.
(Of course, you still need to add it to nixos/release.nix if you want
Hydra to run the test.)

nixos/lib/testing.nix

@@ -73,7 +73,7 @@ rec {
   apply = makeTest; # compatibility
   call = f: f { inherit pkgs system; };
 
-  complete = { testScript, ... } @ t: t // rec {
+  complete = { testScript, makeCoverageReport ? false, ... } @ t: t // rec {
 
     nodes = buildVirtualNetwork (
       t.nodes or (if t ? machine then { machine = t.machine; } else { }));
@@ -117,6 +117,8 @@ rec {
     test = runTests driver;
 
     report = releaseTools.gcovReport { coverageRuns = [ test ]; };
+
+    result = if makeCoverageReport then report else test;
   };
 
 

nixos/release.nix

@@ -14,6 +14,8 @@ let
 
   forAllSystems = pkgs.lib.genAttrs systems;
 
+  callTest = fn: args: forAllSystems (system: (import fn ({ inherit system; } // args)).result);
+
   pkgs = import nixpkgs { system = "x86_64-linux"; };
 
   lib = pkgs.lib;
@@ -207,14 +209,42 @@ in rec {
   */
 
 
-  # Run the tests in ./tests/default.nix for each platform.  You can
-  # run a test by doing e.g. "nix-build -A tests.login.x86_64-linux".
-  tests =
-    with lib;
-    let
-      testsFor = system:
-        mapAttrsRecursiveCond (x: !x ? test)
-          (n: v: listToAttrs [(nameValuePair system (if v.makeCoverageReport or false then v.report else v.test))])
-          (import ./tests { inherit nixpkgs system; });
-    in fold recursiveUpdate {} (map testsFor systems);
+  # Run the tests for each platform.  You can run a test by doing
+  # e.g. ‘nix-build -A tests.login.x86_64-linux’, or equivalently,
+  # ‘nix-build tests/login.nix -A result’.
+  tests.avahi = callTest tests/avahi.nix {};
+  tests.bittorrent = callTest tests/bittorrent.nix {};
+  tests.containers = callTest tests/containers.nix {};
+  tests.firefox = callTest tests/firefox.nix {};
+  tests.firewall = callTest tests/firewall.nix {};
+  tests.gnome3 = callTest tests/gnome3.nix {};
+  tests.installer.grub1 = forAllSystems (system: (import tests/installer.nix { inherit system; }).grub1.test);
+  tests.installer.lvm = forAllSystems (system: (import tests/installer.nix { inherit system; }).lvm.test);
+  tests.installer.rebuildCD = forAllSystems (system: (import tests/installer.nix { inherit system; }).rebuildCD.test);
+  tests.installer.separateBoot = forAllSystems (system: (import tests/installer.nix { inherit system; }).separateBoot.test);
+  tests.installer.simple = forAllSystems (system: (import tests/installer.nix { inherit system; }).simple.test);
+  tests.ipv6 = callTest tests/ipv6.nix {};
+  tests.jenkins = callTest tests/jenkins.nix {};
+  tests.kde4 = callTest tests/kde4.nix {};
+  tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
+  tests.login = callTest tests/login.nix {};
+  tests.logstash = callTest tests/logstash.nix {};
+  tests.misc = callTest tests/misc.nix {};
+  tests.mumble = callTest tests/mumble.nix {};
+  tests.munin = callTest tests/munin.nix {};
+  tests.mysql = callTest tests/mysql.nix {};
+  tests.mysqlReplication = callTest tests/mysql-replication.nix {};
+  tests.nat = callTest tests/nat.nix {};
+  tests.nfs3 = callTest tests/nfs.nix { version = 3; };
+  tests.openssh = callTest tests/openssh.nix {};
+  tests.printing = callTest tests/printing.nix {};
+  tests.proxy = callTest tests/proxy.nix {};
+  tests.quake3 = callTest tests/quake3.nix {};
+  tests.rabbitmq = callTest tests/rabbitmq.nix {};
+  tests.runInMachine = callTest tests/run-in-machine.nix {};
+  tests.simple = callTest tests/simple.nix {};
+  tests.tomcat = callTest tests/tomcat.nix {};
+  tests.udisks = callTest tests/udisks.nix {};
+  tests.xfce = callTest tests/xfce.nix {};
+
 }

nixos/tests/avahi.nix

@@ -1,8 +1,7 @@
-{ pkgs, ... }:
+# Test whether `avahi-daemon' and `libnss-mdns' work as expected.
 
-with pkgs;
+import ./make-test.nix {
 
-{
   nodes = {
     one =
       { config, pkgs, ... }: {
@@ -17,7 +16,6 @@ with pkgs;
       };
   };
 
-  # Test whether `avahi-daemon' and `libnss-mdns' work as expected.
   testScript =
     '' startAll;
 

nixos/tests/bittorrent.nix

@@ -6,7 +6,7 @@
 # which only works if the first client successfully uses the UPnP-IGD
 # protocol to poke a hole in the NAT.
 
-{ pkgs, ... }:
+import ./make-test.nix ({ pkgs, ... }:
 
 let
 
@@ -108,4 +108,4 @@ in
       $client2->succeed("cmp /tmp/test.tar.bz2 ${file}");
     '';
 
-}
+})

nixos/tests/containers.nix

@@ -1,8 +1,6 @@
 # Test for NixOS' container support.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   machine =
     { config, pkgs, ... }:

nixos/tests/default.nix

@@ -1,46 +0,0 @@
-{ nixpkgs ? <nixpkgs>
-, system ? builtins.currentSystem
-, minimal ? false
-}:
-
-with import ../lib/testing.nix { inherit system minimal; };
-
-{
-  avahi = makeTest (import ./avahi.nix);
-  bittorrent = makeTest (import ./bittorrent.nix);
-  containers = makeTest (import ./containers.nix);
-  firefox = makeTest (import ./firefox.nix);
-  firewall = makeTest (import ./firewall.nix);
-  installer = makeTests (import ./installer.nix);
-  efi-installer = makeTests (import ./efi-installer.nix);
-  gnome3 = makeTest (import ./gnome3.nix);
-  ipv6 = makeTest (import ./ipv6.nix);
-  jenkins = makeTest (import ./jenkins.nix);
-  kde4 = makeTest (import ./kde4.nix);
-  #kexec = makeTest (import ./kexec.nix);
-  login = makeTest (import ./login.nix {});
-  logstash = makeTest (import ./logstash.nix);
-  latestKernel.login = makeTest (import ./login.nix ({ config, pkgs, ... }: { boot.kernelPackages = pkgs.linuxPackages_latest; }));
-  misc = makeTest (import ./misc.nix);
-  #mpich = makeTest (import ./mpich.nix);
-  mysql = makeTest (import ./mysql.nix);
-  mysql_replication = makeTest (import ./mysql-replication.nix);
-  munin = makeTest (import ./munin.nix);
-  mumble = makeTest (import ./mumble.nix);
-  nat = makeTest (import ./nat.nix);
-  nfs3 = makeTest (import ./nfs.nix { version = 3; });
-  #nfs4 = makeTest (import ./nfs.nix { version = 4; });
-  openssh = makeTest (import ./openssh.nix);
-  #partition = makeTest (import ./partition.nix);
-  printing = makeTest (import ./printing.nix);
-  proxy = makeTest (import ./proxy.nix);
-  quake3 = makeTest (import ./quake3.nix);
-  rabbitmq = makeTest (import ./rabbitmq.nix);
-  simple = makeTest (import ./simple.nix);
-  #subversion = makeTest (import ./subversion.nix);
-  tomcat = makeTest (import ./tomcat.nix);
-  udisks = makeTest (import ./udisks.nix);
-  #trac = makeTest (import ./trac.nix);
-  xfce = makeTest (import ./xfce.nix);
-  runInMachine.test = import ./run-in-machine.nix { inherit system; };
-}

nixos/tests/firefox.nix

@@ -1,6 +1,4 @@
-{ pkgs, ... }:
-
-{
+import ./make-test.nix ({ pkgs, ... }: {
 
   machine =
     { config, pkgs, ... }:
@@ -18,4 +16,4 @@
       $machine->screenshot("screen");
     '';
 
-}
+})

nixos/tests/firewall.nix

@@ -1,8 +1,6 @@
 # Test the firewall module.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   nodes =
     { walled =

nixos/tests/gnome3.nix

@@ -1,6 +1,4 @@
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   machine =
     { config, pkgs, ... }:

nixos/tests/installer.nix

@@ -1,7 +1,8 @@
-{ pkgs, system, ... }:
+{ system ? builtins.currentSystem }:
 
-with pkgs.lib;
+with import ../lib/testing.nix { inherit system; };
 with import ../lib/qemu-flags.nix;
+with pkgs.lib;
 
 let
 
@@ -99,7 +100,7 @@ let
       my $machine = createMachine({ hda => "harddisk",
         hdaInterface => "${iface}",
         cdrom => glob("${iso}/iso/*.iso"),
-        qemuFlags => '${optionalString testChannel (toString (qemuNICFlags 1 1 2))} ${optionalString (pkgs.stdenv.system == "x86_64-linux") "-cpu kvm64"}'});
+        qemuFlags => '${optionalString testChannel (toString (qemuNICFlags 1 1 2))} ${optionalString (iso.system == "x86_64-linux") "-cpu kvm64"}'});
       $machine->start;
 
       ${optionalString testChannel ''
@@ -190,13 +191,15 @@ let
     '';
 
 
-  makeTest = { createPartitions, fileSystems, testChannel ? false, grubVersion ? 2, grubDevice ? "/dev/vda" }:
-    { inherit iso;
+  makeInstallerTest =
+    { createPartitions, fileSystems, testChannel ? false, grubVersion ? 2, grubDevice ? "/dev/vda" }:
+    makeTest ({ ... }: {
+      inherit iso;
       nodes = if testChannel then { inherit webserver; } else { };
       testScript = testScriptFun {
         inherit createPartitions fileSystems testChannel grubVersion grubDevice;
       };
-    };
+    });
 
 
 in {
@@ -206,7 +209,7 @@ in {
 
   # The (almost) simplest partitioning scheme: a swap partition and
   # one big filesystem partition.
-  simple = makeTest
+  simple = makeInstallerTest
     { createPartitions =
         ''
           $machine->succeed(
@@ -225,7 +228,7 @@ in {
     };
 
   # Same as the previous, but now with a separate /boot partition.
-  separateBoot = makeTest
+  separateBoot = makeInstallerTest
     { createPartitions =
         ''
           $machine->succeed(
@@ -248,7 +251,7 @@ in {
 
   # Create two physical LVM partitions combined into one volume group
   # that contains the logical swap and root partitions.
-  lvm = makeTest
+  lvm = makeInstallerTest
     { createPartitions =
         ''
           $machine->succeed(
@@ -271,8 +274,7 @@ in {
       fileSystems = rootFS;
     };
 
-  /*
-  swraid = makeTest
+  swraid = makeInstallerTest
     { createPartitions =
         ''
           $machine->succeed(
@@ -304,10 +306,9 @@ in {
         '';
       fileSystems = rootFS + bootFS;
     };
-  */
 
   # Test a basic install using GRUB 1.
-  grub1 = makeTest
+  grub1 = makeInstallerTest
     { createPartitions =
         ''
           $machine->succeed(
@@ -328,7 +329,7 @@ in {
     };
 
   # Rebuild the CD configuration with a little modification.
-  rebuildCD =
+  rebuildCD = makeTest ({ ... }:
     { inherit iso;
       nodes = { };
       testScript =
@@ -352,5 +353,5 @@ in {
 
           $machine->shutdown;
         '';
-    };
+    });
 }

nixos/tests/ipv6.nix

@@ -1,9 +1,7 @@
 # Test of IPv6 functionality in NixOS, including whether router
 # solicication/advertisement using radvd works.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   nodes =
     { client = { config, pkgs, ... }: { };

nixos/tests/jenkins.nix

@@ -2,9 +2,9 @@
 #   1. jenkins service starts on master node
 #   2. jenkins user can be extended on both master and slave
 #   3. jenkins service not started on slave node
-{ pkgs, ... }:
 
-{
+import ./make-test.nix {
+
   nodes = {
 
     master =

nixos/tests/kde4.nix

@@ -1,6 +1,4 @@
-{ pkgs, ... }:
-
-{
+import ./make-test.nix ({ pkgs, ... }: {
 
   machine =
     { config, pkgs, ... }:
@@ -64,4 +62,4 @@
       $machine->screenshot("screen");
     '';
 
-}
+})

nixos/tests/kexec.nix

@@ -1,8 +1,6 @@
 # Test whether fast reboots via kexec work.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix  {
 
   machine = { config, pkgs, ... }:
     { virtualisation.vlans = [ ]; };

nixos/tests/login.nix

@@ -1,8 +1,11 @@
-config: { pkgs, ... }:
+import ./make-test.nix ({ pkgs, latestKernel ? false, ... }:
 
 {
 
-  machine = config;
+  machine =
+    { config, pkgs, lib, ... }:
+    { boot.kernelPackages = lib.mkIf latestKernel pkgs.linuxPackages_latest;
+    };
 
   testScript =
     ''
@@ -58,4 +61,4 @@ config: { pkgs, ... }:
       };
     '';
 
-}
+})

nixos/tests/logstash.nix

@@ -1,9 +1,8 @@
-{ pkgs, ... }:
+# This test runs logstash and checks if messages flows and
+# elasticsearch is started.
 
-# This test runs logstash and checks if messages flows and elasticsearch is
-# started
+import ./make-test.nix {
 
-{
   nodes = {
     one =
       { config, pkgs, ... }:
@@ -28,10 +27,10 @@
           };
         };
     };
-  
+
   testScript = ''
     startAll;
-  
+
     $one->waitForUnit("logstash.service");
     $one->waitUntilSucceeds("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep flowers");
     $one->fail("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep dragons");

nixos/tests/misc.nix

@@ -1,8 +1,6 @@
 # Miscellaneous small tests that don't warrant their own VM run.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   machine =
     { config, pkgs, ... }:

nixos/tests/mpich.nix

@@ -1,10 +1,6 @@
 # Simple example to showcase distributed tests using NixOS VMs.
 
-{ pkgs, ... }:
-
-with pkgs;
-
-{
+import ./make-test.nix {
   nodes = {
     master =
       { config, pkgs, ... }: {

nixos/tests/mumble.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+import ./make-test.nix (
 
 let
   client = { config, pkgs, ... }: {
@@ -52,4 +52,4 @@ in
     $client1->screenshot("screen1");
     $client2->screenshot("screen2");
   '';
-}
+})

nixos/tests/munin.nix

@@ -1,13 +1,12 @@
-{ pkgs, ... }:
-
 # This test runs basic munin setup with node and cron job running on the same
 # machine.
 
-{
-  nodes = { 
+import ./make-test.nix {
+
+  nodes = {
     one =
       { config, pkgs, ... }:
-        { 
+        {
           services = {
            munin-node.enable = true;
            munin-cron = {
@@ -20,10 +19,10 @@
           };
         };
     };
-  
+
   testScript = ''
     startAll;
-  
+
     $one->waitForUnit("munin-node.service");
     $one->waitForFile("/var/lib/munin/one/one-uptime-uptime-g.rrd");
     $one->waitForFile("/var/www/munin/one/index.html");

nixos/tests/mysql-replication.nix

@@ -1,9 +1,10 @@
-{ pkgs, ... }:
+import ./make-test.nix (
 
 let
   replicateUser = "replicate";
   replicatePassword = "secret";
 in
+
 {
   nodes = {
     master =
@@ -58,4 +59,4 @@ in
     $slave2->sleep(100); # Hopefully this is long enough!!
     $slave2->succeed("echo 'use testdb; select * from tests' | mysql -u root -N | grep 4");
   '';
-}
+})

nixos/tests/mysql.nix

@@ -1,6 +1,5 @@
-{ pkgs, ... }:
+import ./make-test.nix {
 
-{
   nodes = {
     master =
       { pkgs, config, ... }:

nixos/tests/nat.nix

@@ -4,9 +4,7 @@
 # router connected to both that performs Network Address Translation
 # for the client.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   nodes =
     { client =

nixos/tests/nfs.nix

@@ -1,6 +1,4 @@
-{ version }:
-
-{ pkgs, ... }:
+import ./make-test.nix ({ version, ... }:
 
 let
 
@@ -84,4 +82,4 @@ in
       die "shutdown took too long ($duration seconds)" if $duration > 30;
     '';
 
-}
+})

nixos/tests/openssh.nix

@@ -1,6 +1,5 @@
-{ pkgs, ... }:
+import ./make-test.nix ({ pkgs, ... }: {
 
-{
   nodes = {
 
     server =
@@ -35,4 +34,4 @@
     $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'echo hello world' >&2");
     $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'ulimit -l' | grep 1024");
   '';
-}
+})

nixos/tests/partition.nix

@@ -1,4 +1,4 @@
-{ pkgs, system, ... }:
+import ./make-test.nix ({ pkgs, ... }:
 
 with pkgs.lib;
 
@@ -224,4 +224,4 @@ in {
       ensureMountPoint("/mnt/boot");
     };
   '';
-}
+})

nixos/tests/printing.nix

@@ -1,8 +1,6 @@
 # Test printing via CUPS.
 
-{ pkgs, ... }:
-
-{
+import ./make-test.nix ({pkgs, ... }: {
 
   nodes = {
 
@@ -88,4 +86,4 @@
       }
     '';
 
-}
+})

nixos/tests/proxy.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+import ./make-test.nix (
 
 let
 
@@ -90,4 +90,4 @@ in
       $client->succeed("curl --fail http://proxy/");
     '';
 
-}
+})

nixos/tests/quake3.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+import ./make-test.nix (
 
 let
 
@@ -79,4 +79,4 @@ rec {
       $server->stopJob("quake3-server");
     '';
 
-}
+})

nixos/tests/rabbitmq.nix

@@ -1,8 +1,7 @@
-{ pkgs, ... }:
+# This test runs rabbitmq and checks if rabbitmq is up and running.
 
-# This test runs rabbitmq and checks if rabbitmq is up and running
+import ./make-test.nix ({ pkgs, ... }: {
 
-{
   nodes = {
     one = { config, pkgs, ... }: {
       services.rabbitmq.enable = true;
@@ -11,8 +10,8 @@
 
   testScript = ''
     startAll;
-  
+
     $one->waitForUnit("rabbitmq.service");
     $one->waitUntilSucceeds("su -s ${pkgs.stdenv.shell} rabbitmq -c \"rabbitmqctl status\"");
   '';
-}
+})

nixos/tests/run-in-machine.nix

@@ -2,7 +2,9 @@
 
 with import ../lib/testing.nix { inherit system; };
 
-runInMachine {
-  drv = pkgs.patchelf;
-  machine = { config, pkgs, ... }: { services.sshd.enable = true; };
+{
+  test = runInMachine {
+    drv = pkgs.hello;
+    machine = { config, pkgs, ... }: { /* services.sshd.enable = true; */ };
+  };
 }

nixos/tests/simple.nix

@@ -1,11 +1,11 @@
-{ pkgs, ... }:
+import ./make-test.nix {
 
-{
   machine = { config, pkgs, ... }: { };
 
   testScript =
     ''
       startAll;
+      $machine->waitForUnit("multi-user.target");
       $machine->shutdown;
     '';
 }

nixos/tests/subversion.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+import ./make-test.nix (
 
 let
 
@@ -114,4 +114,4 @@ in
       $webserver->stopJob("httpd");
     '';
 
-}
+})

nixos/tests/tomcat.nix

@@ -1,6 +1,5 @@
-{ pkgs, ... }:
+import ./make-test.nix {
 
-{
   nodes = {
     server =
       { pkgs, config, ... }:
@@ -25,4 +24,5 @@
     $client->succeed("curl --fail http://server/examples/servlets/servlet/HelloWorldExample");
     $client->succeed("curl --fail http://server/examples/jsp/jsp2/simpletag/hello.jsp");
   '';
+
 }

nixos/tests/trac.nix

@@ -1,6 +1,5 @@
-{ pkgs, ... }:
+import ./make-test.nix ({ pkgs, ... }: {
 
-{
   nodes = {
     storage =
       { config, pkgs, ... }:
@@ -68,4 +67,4 @@
 
       $client->screenshot("screen");
     '';
-}
+})

nixos/tests/udisks.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+import ./make-test.nix ({ pkgs, ... }:
 
 let
 
@@ -53,4 +53,4 @@ in
       $machine->fail("[ -e /dev/sda ]");
     '';
 
-}
+})

nixos/tests/xfce.nix

@@ -1,6 +1,4 @@
-{ pkgs, ... }:
-
-{
+import ./make-test.nix {
 
   machine =
     { config, pkgs, ... }: