Merge pull request #44439 from Ekleog/meta-tests

[RFC] Use `meta.tests` to link from packages to the tests that test them

doc/meta.xml

@@ -250,6 +250,61 @@ meta.platforms = stdenv.lib.platforms.linux;
      </para>
     </listitem>
    </varlistentry>
+   <varlistentry>
+    <term>
+     <varname>tests</varname>
+    </term>
+    <listitem>
+     <para>
+      An attribute set with as values tests. A test is a derivation, which
+      builds successfully when the test passes, and fails to build otherwise. A
+      derivation that is a test requires some <literal>meta</literal> elements
+      to be defined: <literal>needsVMSupport</literal> (automatically filled-in
+      for NixOS tests) and <literal>timeout</literal>.
+     </para>
+     <para>
+      The NixOS tests are available as <literal>nixosTests</literal> in
+      parameters of derivations. For instance, the OpenSMTPD derivation
+      includes lines similar to:
+<programlisting>
+{ /* ... */, nixosTests }:
+{
+  # ...
+  meta.tests = {
+    basic-functionality-and-dovecot-integration = nixosTests.opensmtpd;
+  };
+}
+</programlisting>
+     </para>
+    </listitem>
+   </varlistentry>
+   <varlistentry>
+    <term>
+     <varname>timeout</varname>
+    </term>
+    <listitem>
+     <para>
+      A timeout (in seconds) for building the derivation. If the derivation
+      takes longer than this time to build, it can fail due to breaking the
+      timeout. However, all computers do not have the same computing power,
+      hence some builders may decide to apply a multiplicative factor to this
+      value. When filling this value in, try to keep it approximately
+      consistent with other values already present in
+      <literal>nixpkgs</literal>.
+     </para>
+    </listitem>
+   </varlistentry>
+   <varlistentry>
+    <term>
+     <varname>needsVMSupport</varname>
+    </term>
+    <listitem>
+     <para>
+      A boolan that states whether the derivation requires build-time support
+      for Virtual Machine to build successfully.
+     </para>
+    </listitem>
+   </varlistentry>
    <varlistentry>
     <term>
      <varname>hydraPlatforms</varname>

nixos/lib/testing.nix

@@ -69,7 +69,9 @@ in rec {
             mkdir -p $out/coverage-data
             mv $i $out/coverage-data/$(dirname $(dirname $i))
           done
-        ''; # */
+        '';
+
+        meta.needsVMSupport = true;
     };
 
 

nixos/tests/opensmtpd.nix

@@ -120,4 +120,6 @@ import ./make-test.nix {
     $smtp2->waitUntilFails('smtpctl show queue | egrep .');
     $client->succeed('check-mail-landed >&2');
   '';
+
+  meta.timeout = 30;
 }

pkgs/servers/mail/dovecot/default.nix

@@ -1,6 +1,7 @@
 { stdenv, lib, fetchurl, perl, pkgconfig, systemd, openssl
 , bzip2, zlib, lz4, inotify-tools, pam, libcap
 , clucene_core_2, icu, openldap, libsodium, libstemmer, cyrus_sasl
+, nixosTests
 # Auth modules
 , withMySQL ? false, mysql
 , withPgSQL ? false, postgresql
@@ -74,5 +75,8 @@ stdenv.mkDerivation rec {
     description = "Open source IMAP and POP3 email server written with security primarily in mind";
     maintainers = with stdenv.lib.maintainers; [ peti rickynils fpletz ];
     platforms = stdenv.lib.platforms.unix;
+    tests = {
+      opensmtpd-interaction = nixosTests.opensmtpd;
+    };
   };
 }

pkgs/servers/mail/opensmtpd/default.nix

@@ -1,5 +1,5 @@
 { stdenv, lib, fetchurl, fetchpatch, autoconf, automake, libtool, bison
-, libasr, libevent, zlib, libressl, db, pam
+, libasr, libevent, zlib, libressl, db, pam, nixosTests
 }:
 
 stdenv.mkDerivation rec {
@@ -61,5 +61,8 @@ stdenv.mkDerivation rec {
     license = licenses.isc;
     platforms = platforms.linux;
     maintainers = with maintainers; [ rickynils obadz ekleog ];
+    tests = {
+      basic-functionality-and-dovecot-interaction = nixosTests.opensmtpd;
+    };
   };
 }

pkgs/stdenv/generic/check-meta.nix

@@ -165,6 +165,16 @@ let
     platforms = listOf (either str lib.systems.parsedPlatform.types.system);
     hydraPlatforms = listOf str;
     broken = bool;
+    # TODO: refactor once something like Profpatsch's types-simple will land
+    tests = attrsOf (mkOptionType {
+      name = "test";
+      check = x: isDerivation x &&
+        x ? meta.timeout &&
+        x ? meta.needsVMSupport;
+      merge = lib.options.mergeOneOption;
+    });
+    needsVMSupport = bool;
+    timeout = int;
 
     # Weirder stuff that doesn't appear in the documentation?
     knownVulnerabilities = listOf str;
@@ -184,8 +194,6 @@ let
     isIbusEngine = bool;
     isGutenprint = bool;
     badPlatforms = platforms;
-    # Hydra build timeout
-    timeout = int;
   };
 
   checkMetaAttr = k: v:

pkgs/top-level/all-packages.nix

@@ -71,6 +71,26 @@ with pkgs;
 
   common-updater-scripts = callPackage ../common-updater/scripts.nix { };
 
+  ### Push NixOS tests inside the fixed point
+
+  nixosTests =
+    let
+      # TODO(Ericson2314,ekleog): Check this will work correctly with cross-
+      system = builtins.currentSystem;
+      rawTests = (import ../../nixos/release.nix {
+        nixpkgs = pkgs;
+      }).tests;
+      testNames = builtins.attrNames rawTests;
+      filteredList = builtins.filter
+        (test: rawTests.${test} ? ${system})
+        testNames;
+      finalList = map
+        (test: { name = test; value = rawTests.${test}.${system}; })
+        filteredList;
+      finalTests = builtins.listToAttrs finalList;
+    in
+    finalTests;
+
   ### BUILD SUPPORT
 
   autoreconfHook = makeSetupHook