lib: Fix system parsing, and use for doubles lists

The old hard-coded lists are now used to test system parsing.

In the process, make an `assertTrue` in release lib for eval tests; also
use it in release-cross

lib/systems/doubles.nix

@@ -1,24 +1,44 @@
 let lists = import ../lists.nix; in
+let parse = import ./parse.nix; in
+let inherit (import ../attrsets.nix) matchAttrs; in
+
+let
+  all = [
+    "aarch64-linux"
+    "armv5tel-linux" "armv6l-linux" "armv7l-linux"
+
+    "mips64el-linux"
+
+    "i686-cygwin" "i686-freebsd" "i686-linux" "i686-netbsd" "i686-openbsd"
+
+    "x86_64-cygwin" "x86_64-darwin" "x86_64-freebsd" "x86_64-linux"
+    "x86_64-netbsd" "x86_64-openbsd" "x86_64-solaris"
+  ];
+
+  allParsed = map parse.mkSystemFromString all;
+
+  filterDoubles = f: map parse.doubleFromSystem (lists.filter f allParsed);
+
+in rec {
+  inherit all;
 
-rec {
-  all = linux ++ darwin ++ cygwin ++ freebsd ++ openbsd ++ netbsd ++ illumos;
   allBut = platforms: lists.filter (x: !(builtins.elem x platforms)) all;
   none = [];
 
-  arm = ["armv5tel-linux" "armv6l-linux" "armv7l-linux" ];
+  arm = filterDoubles (matchAttrs { cpu = { family = "arm"; bits = 32; }; });
-  i686 = ["i686-linux" "i686-freebsd" "i686-netbsd" "i686-cygwin"];
+  i686 = filterDoubles parse.isi686;
-  mips = [ "mips64el-linux" ];
+  mips = filterDoubles (matchAttrs { cpu = { family = "mips"; }; });
-  x86_64 = ["x86_64-linux" "x86_64-darwin" "x86_64-freebsd" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin"];
+  x86_64 = filterDoubles parse.isx86_64;
 
-  cygwin = ["i686-cygwin" "x86_64-cygwin"];
+  cygwin = filterDoubles (matchAttrs { kernel = parse.kernels.cygwin; });
-  darwin = ["x86_64-darwin"];
+  darwin = filterDoubles parse.isDarwin;
-  freebsd = ["i686-freebsd" "x86_64-freebsd"];
+  freebsd = filterDoubles (matchAttrs { kernel = parse.kernels.freebsd; });
-  gnu = linux; /* ++ hurd ++ kfreebsd ++ ... */
+  gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }); # Should be better
-  illumos = ["x86_64-solaris"];
+  illumos = filterDoubles (matchAttrs { kernel = parse.kernels.solaris; });
-  linux = ["i686-linux" "x86_64-linux" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux" "mips64el-linux"];
+  linux = filterDoubles parse.isLinux;
-  netbsd = ["i686-netbsd" "x86_64-netbsd"];
+  netbsd = filterDoubles (matchAttrs { kernel = parse.kernels.netbsd; });
-  openbsd = ["i686-openbsd" "x86_64-openbsd"];
+  openbsd = filterDoubles (matchAttrs { kernel = parse.kernels.openbsd; });
-  unix = linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos;
+  unix = filterDoubles parse.isUnix;
 
   mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux"];
 }

lib/systems/parse.nix

@@ -6,11 +6,13 @@ with import ../types.nix;
 with import ../attrsets.nix;
 
 let
-  lib = import ./default.nix;
+  lib = import ../default.nix;
-  setTypes = type:
+  setTypesAssert = type: pred:
     mapAttrs (name: value:
-      setType type ({inherit name;} // value)
+      #assert pred value;
-    );
+      setType type ({ inherit name; } // value));
+  setTypes = type: setTypesAssert type (_: true);
+
 in
 
 rec {
@@ -22,20 +24,31 @@ rec {
   };
 
 
-  isCpuType = x: isType "cpu-type" x
+  isCpuType = isType "cpu-type";
-    && elem x.bits [8 16 32 64 128]
+  cpuTypes = with significantBytes; setTypesAssert "cpu-type"
-    && (8 < x.bits -> isSignificantByte x.significantByte);
+    (x: elem x.bits [8 16 32 64 128]
-
+        && (if 8 < x.bits
-  cpuTypes = with significantBytes;
+            then isSignificantByte x.significantByte
-    setTypes "cpu-type" {
+            else !(x ? significantByte)))
-      arm =      { bits = 32; significantByte = littleEndian; };
+  {
-      armv5tel = { bits = 32; significantByte = littleEndian; };
+    arm      = { bits = 32; significantByte = littleEndian; family = "arm"; };
-      armv7l   = { bits = 32; significantByte = littleEndian; };
+    armv5tel = { bits = 32; significantByte = littleEndian; family = "arm"; };
-      i686 =     { bits = 32; significantByte = littleEndian; };
+    armv6l   = { bits = 32; significantByte = littleEndian; family = "arm"; };
-      powerpc =  { bits = 32; significantByte = bigEndian; };
+    armv7a   = { bits = 32; significantByte = littleEndian; family = "arm"; };
-      x86_64 =   { bits = 64; significantByte = littleEndian; };
+    armv7l   = { bits = 32; significantByte = littleEndian; family = "arm"; };
-    };
+    aarch64  = { bits = 64; significantByte = littleEndian; family = "arm"; };
+    i686     = { bits = 32; significantByte = littleEndian; family = "x86"; };
+    x86_64   = { bits = 64; significantByte = littleEndian; family = "x86"; };
+    mips64el = { bits = 32; significantByte = littleEndian; family = "mips"; };
+    powerpc  = { bits = 32; significantByte = bigEndian;    family = "powerpc"; };
+  };
 
+  isVendor = isType "vendor";
+  vendors = setTypes "vendor" {
+    apple = {};
+    pc = {};
+    unknown = {};
+  };
 
   isExecFormat = isType "exec-format";
   execFormats = setTypes "exec-format" {
@@ -43,84 +56,118 @@ rec {
     elf = {};
     macho = {};
     pe = {};
-    unknow = {};
+    unknown = {};
+  };
+
+  isKernelFamily = isType "kernel-family";
+  kernelFamilies = setTypes "kernel-family" {
+    bsd = {};
+    unix = {};
+    windows-nt = {};
+    dos = {};
+  };
+
+  isKernel = x: isType "kernel" x;
+  kernels = with execFormats; with kernelFamilies; setTypesAssert "kernel"
+    (x: isExecFormat x.execFormat && all isKernelFamily (attrValues x.families))
+  {
+    cygwin  = { execFormat = pe;      families = { inherit /*unix*/ windows-nt; }; };
+    darwin  = { execFormat = macho;   families = { inherit unix; }; };
+    freebsd = { execFormat = elf;     families = { inherit unix bsd; }; };
+    linux   = { execFormat = elf;     families = { inherit unix; }; };
+    netbsd  = { execFormat = elf;     families = { inherit unix bsd; }; };
+    none    = { execFormat = unknown; families = { inherit unix; }; };
+    openbsd = { execFormat = elf;     families = { inherit unix bsd; }; };
+    solaris = { execFormat = elf;     families = { inherit unix; }; };
+    win32   = { execFormat = pe;      families = { inherit dos; }; };
   };
 
 
-  isKernel = isType "kernel";
+  isAbi = isType "abi";
-  kernels = with execFormats;
+  abis = setTypes "abi" {
-    setTypes "kernel" {
+    gnu = {};
-      cygwin =  { execFormat = pe; };
+    msvc = {};
-      darwin =  { execFormat = macho; };
+    eabi = {};
-      freebsd = { execFormat = elf; };
+    androideabi = {};
-      linux =   { execFormat = elf; };
+    unknown = {};
-      netbsd =  { execFormat = elf; };
+  };
-      none =    { execFormat = unknow; };
+
-      openbsd = { execFormat = elf; };
+  isSystem = isType "system";
-      win32 =   { execFormat = pe; };
+  mkSystem = { cpu, vendor, kernel, abi }:
+    assert isCpuType cpu && isVendor vendor && isKernel kernel && isAbi abi;
+    setType "system" {
+      inherit cpu vendor kernel abi;
     };
 
-
-  isArchitecture = isType "architecture";
-  architectures = setTypes "architecture" {
-    apple = {};
-    pc = {};
-    unknow = {};
-  };
-
-
-  isSystem = x: isType "system" x
-    && isCpuType x.cpu
-    && isArchitecture x.arch
-    && isKernel x.kernel;
-
-  mkSystem = {
-    cpu ? cpuTypes.i686,
-    arch ? architectures.pc,
-    kernel ? kernels.linux,
-    name ? "${cpu.name}-${arch.name}-${kernel.name}"
-  }: setType "system" {
-    inherit name cpu arch kernel;
-  };
-
-
   is64Bit = matchAttrs { cpu = { bits = 64; }; };
-  isDarwin = matchAttrs { kernel = kernels.darwin; };
+  is32Bit = matchAttrs { cpu = { bits = 32; }; };
   isi686 = matchAttrs { cpu = cpuTypes.i686; };
-  isLinux = matchAttrs { kernel = kernels.linux; };
+  isx86_64 = matchAttrs { cpu = cpuTypes.x86_64; };
 
+  isDarwin = matchAttrs { kernel = kernels.darwin; };
+  isLinux = matchAttrs { kernel = kernels.linux; };
+  isUnix = matchAttrs { kernel = { families = { inherit (kernelFamilies) unix; }; }; };
+  isWindows = s: matchAttrs { kernel = { families = { inherit (kernelFamilies) windows-nt; }; }; } s
+              || matchAttrs { kernel = { families = { inherit (kernelFamilies) dos; }; }; } s;
+
+
+  mkSkeletonFromList = l: {
+    "2" =    { cpu = elemAt l 0;                      kernel = elemAt l 1;                   };
+    "4" =    { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; abi = elemAt l 3; };
+    "3" = # Awkwards hacks, beware!
+      if elemAt l 1 == "apple"
+        then { cpu = elemAt l 0; vendor = "apple";    kernel = elemAt l 2;                   }
+      else if (elemAt l 1 == "linux") || (elemAt l 2 == "gnu")
+        then { cpu = elemAt l 0;                      kernel = elemAt l 1; abi = elemAt l 2; }
+      else throw "Target specification with 3 components is ambiguous";
+  }.${toString (length l)}
+    or (throw "system string has invalid number of hyphen-separated components");
 
   # This should revert the job done by config.guess from the gcc compiler.
-  mkSystemFromString = s: let
+  mkSystemFromSkeleton = { cpu
-    l = lib.splitString "-" s;
+                         , # Optional, but fallback too complex for here.
-
+                           # Inferred below instead.
+                           vendor ? assert false; null
+                         , kernel
+                         , # Also inferred below
+                           abi    ? assert false; null
+                         } @ args: let
     getCpu = name:
-      attrByPath [name] (throw "Unknow cpuType `${name}'.")
+      attrByPath [name] (throw "Unknown CPU type: ${name}")
         cpuTypes;
-    getArch = name:
+    getVendor = name:
-      attrByPath [name] (throw "Unknow architecture `${name}'.")
+      attrByPath [name] (throw "Unknown vendor: ${name}")
-        architectures;
+        vendors;
     getKernel = name:
-      attrByPath [name] (throw "Unknow kernel `${name}'.")
+      attrByPath [name] (throw "Unknown kernel: ${name}")
         kernels;
+    getAbi = name:
+      attrByPath [name] (throw "Unknown ABI: ${name}")
+        abis;
+
+    system = rec {
+      cpu = getCpu args.cpu;
+      vendor =
+        /**/ if args ? vendor    then getVendor args.vendor
+        else if isDarwin  system then vendors.apple
+        else if isWindows system then vendors.pc
+        else                     vendors.unknown;
+      kernel = getKernel args.kernel;
+      abi =
+        /**/ if args ? abi       then getAbi args.abi
+        else if isLinux   system then abis.gnu
+        else if isWindows system then abis.gnu
+        else                     abis.unknown;
+    };
+
+  in mkSystem system;
+
+  mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s));
+
+  doubleFromSystem = { cpu, vendor, kernel, abi, ... }: "${cpu.name}-${kernel.name}";
+
+  tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let
+    optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}";
+  in "${cpu.name}-${vendor.name}-${kernel.name}${optAbi}";
 
-    system =
-      if builtins.length l == 2 then
-        mkSystem rec {
-          name = s;
-          cpu = getCpu (head l);
-          arch =
-            if isDarwin system
-            then architectures.apple
-            else architectures.pc;
-          kernel = getKernel (head (tail l));
-        }
-      else
-        mkSystem {
-          name = s;
-          cpu = getCpu (head l);
-          arch = getArch (head (tail l));
-          kernel = getKernel (head (tail (tail l)));
-        };
-  in assert isSystem system; system;
 }

lib/tests/release.nix

@@ -1,31 +1,40 @@
-{ nixpkgs }:
+{ nixpkgs ? { outPath = (import ../.).cleanSource ../..; revCount = 1234; shortRev = "abcdef"; }
+, # The platforms for which we build Nixpkgs.
+  supportedSystems ? [ builtins.currentSystem ]
+, # Strip most of attributes when evaluating to spare memory usage
+  scrubJobs ? true
+}:
 
-with import ../.. { };
+with import ../../pkgs/top-level/release-lib.nix { inherit supportedSystems scrubJobs; };
 with lib;
 
-stdenv.mkDerivation {
+{
-  name = "nixpkgs-lib-tests";
+  systems = import ./systems.nix { inherit lib assertTrue; };
-  buildInputs = [ nix ];
-  NIX_PATH="nixpkgs=${nixpkgs}";
 
-  buildCommand = ''
+  moduleSystem = pkgs.stdenv.mkDerivation {
-    datadir="${nix}/share"
+    name = "nixpkgs-lib-tests";
-    export TEST_ROOT=$(pwd)/test-tmp
+    buildInputs = [ pkgs.nix ];
-    export NIX_BUILD_HOOK=
+    NIX_PATH="nixpkgs=${nixpkgs}";
-    export NIX_CONF_DIR=$TEST_ROOT/etc
-    export NIX_DB_DIR=$TEST_ROOT/db
-    export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
-    export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
-    export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
-    export NIX_STATE_DIR=$TEST_ROOT/var/nix
-    export NIX_STORE_DIR=$TEST_ROOT/store
-    export PAGER=cat
-    cacheDir=$TEST_ROOT/binary-cache
-    nix-store --init
 
-    cd ${nixpkgs}/lib/tests
+    buildCommand = ''
-    ./modules.sh
+      datadir="${pkgs.nix}/share"
+      export TEST_ROOT=$(pwd)/test-tmp
+      export NIX_BUILD_HOOK=
+      export NIX_CONF_DIR=$TEST_ROOT/etc
+      export NIX_DB_DIR=$TEST_ROOT/db
+      export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
+      export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
+      export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
+      export NIX_STATE_DIR=$TEST_ROOT/var/nix
+      export NIX_STORE_DIR=$TEST_ROOT/store
+      export PAGER=cat
+      cacheDir=$TEST_ROOT/binary-cache
+      nix-store --init
 
-    touch $out
+      cd ${nixpkgs}/lib/tests
-  '';
+      ./modules.sh
+
+      touch $out
+    '';
+  };
 }

lib/tests/systems.nix

@@ -0,0 +1,31 @@
+# We assert that the new algorithmic way of generating these lists matches the
+# way they were hard-coded before.
+#
+# One might think "if we exhaustively test, what's the point of procedurally
+# calculating the lists anyway?". The answer is one can mindlessly update these
+# tests as new platforms become supported, and then just give the diff a quick
+# sanity check before committing :).
+{ lib, assertTrue }:
+
+with lib.systems.doubles;
+
+let mseteq = x: y: lib.sort lib.lessThan x == lib.sort lib.lessThan y; in
+
+{
+  all = assertTrue (mseteq all (linux ++ darwin ++ cygwin ++ freebsd ++ openbsd ++ netbsd ++ illumos));
+
+  arm = assertTrue (mseteq arm [ "armv5tel-linux" "armv6l-linux" "armv7l-linux" ]);
+  i686 = assertTrue (mseteq i686 [ "i686-linux" "i686-freebsd" "i686-netbsd" "i686-cygwin" ]);
+  mips = assertTrue (mseteq mips [ "mips64el-linux" ]);
+  x86_64 = assertTrue (mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" ]);
+
+  cygwin = assertTrue (mseteq cygwin [ "i686-cygwin" "x86_64-cygwin" ]);
+  darwin = assertTrue (mseteq darwin [ "x86_64-darwin" ]);
+  freebsd = assertTrue (mseteq freebsd [ "i686-freebsd" "x86_64-freebsd" ]);
+  gnu = assertTrue (mseteq gnu (linux /* ++ hurd ++ kfreebsd ++ ... */));
+  illumos = assertTrue (mseteq illumos [ "x86_64-solaris" ]);
+  linux = assertTrue (mseteq linux [ "i686-linux" "x86_64-linux" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux" "mips64el-linux" ]);
+  netbsd = assertTrue (mseteq netbsd [ "i686-netbsd" "x86_64-netbsd" ]);
+  openbsd = assertTrue (mseteq openbsd [ "i686-openbsd" "x86_64-openbsd" ]);
+  unix = assertTrue (mseteq unix (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos));
+}

pkgs/top-level/release-cross.nix

@@ -47,11 +47,11 @@ in
     # cause false negatives.
     testEqualOne = path: system: let
       f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs));
-    in assert
+    in assertTrue (
         f path { inherit system; }
         ==
         f (["buildPackages"] ++ path) { inherit system crossSystem; };
-      true;
+      );
 
     testEqual = path: systems: forAllSupportedSystems systems (testEqualOne path);
 

pkgs/top-level/release-lib.nix

@@ -46,6 +46,11 @@ rec {
   pkgs_x86_64_cygwin = allPackages { system = "x86_64-cygwin"; };
 
 
+  assertTrue = bool:
+    if bool
+    then pkgs.runCommand "evaluated-to-true" {} "touch $out"
+    else pkgs.runCommand "evaluated-to-false" {} "false";
+
   /* The working or failing mails for cross builds will be sent only to
      the following maintainers, as most package maintainers will not be
      interested in the result of cross building a package. */

pkgs/top-level/release.nix

@@ -26,7 +26,7 @@ let
       metrics = import ./metrics.nix { inherit pkgs nixpkgs; };
 
       manual = import ../../doc;
-      lib-tests = import ../../lib/tests/release.nix { inherit nixpkgs; };
+      lib-tests = import ../../lib/tests/release.nix { inherit nixpkgs supportedSystems scrubJobs; };
 
       unstable = pkgs.releaseTools.aggregate
         { name = "nixpkgs-${jobs.tarball.version}";
@@ -35,7 +35,6 @@ let
             [ jobs.tarball
               jobs.metrics
               jobs.manual
-              jobs.lib-tests
               jobs.stdenv.x86_64-linux
               jobs.stdenv.i686-linux
               jobs.stdenv.x86_64-darwin
@@ -62,7 +61,8 @@ let
               jobs.git.x86_64-darwin
               jobs.mysql.x86_64-darwin
               jobs.vim.x86_64-darwin
-            ] ++ lib.collect lib.isDerivation jobs.stdenvBootstrapTools;
+            ] ++ lib.collect lib.isDerivation jobs.stdenvBootstrapTools
+              ++ lib.collect lib.isDerivation jobs.lib-tests;
         };
     } // (lib.optionalAttrs (builtins.elem "i686-linux" supportedSystems) {
       stdenvBootstrapTools.i686-linux =