From 8c99aab3ea4a9ddbd3918282da7861e674b4e7c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Feb 2017 16:09:47 -0500 Subject: [PATCH] lib: Fix system parsing, and use for doubles lists The old hard-coded lists are now used to test system parsing. In the process, make an `assertTrue` in release lib for eval tests; also use it in release-cross --- lib/systems/doubles.nix | 50 +++++--- lib/systems/parse.nix | 211 +++++++++++++++++++------------ lib/tests/release.nix | 57 +++++---- lib/tests/systems.nix | 31 +++++ pkgs/top-level/release-cross.nix | 4 +- pkgs/top-level/release-lib.nix | 5 + pkgs/top-level/release.nix | 6 +- 7 files changed, 238 insertions(+), 126 deletions(-) create mode 100644 lib/tests/systems.nix diff --git a/lib/systems/doubles.nix b/lib/systems/doubles.nix index 945147fe7cfb..2622ddf4be1f 100644 --- a/lib/systems/doubles.nix +++ b/lib/systems/doubles.nix @@ -1,24 +1,44 @@ let lists = import ../lists.nix; in +let parse = import ./parse.nix; in +let inherit (import ../attrsets.nix) matchAttrs; in + +let + all = [ + "aarch64-linux" + "armv5tel-linux" "armv6l-linux" "armv7l-linux" + + "mips64el-linux" + + "i686-cygwin" "i686-freebsd" "i686-linux" "i686-netbsd" "i686-openbsd" + + "x86_64-cygwin" "x86_64-darwin" "x86_64-freebsd" "x86_64-linux" + "x86_64-netbsd" "x86_64-openbsd" "x86_64-solaris" + ]; + + allParsed = map parse.mkSystemFromString all; + + filterDoubles = f: map parse.doubleFromSystem (lists.filter f allParsed); + +in rec { + inherit all; -rec { - all = linux ++ darwin ++ cygwin ++ freebsd ++ openbsd ++ netbsd ++ illumos; allBut = platforms: lists.filter (x: !(builtins.elem x platforms)) all; none = []; - arm = ["armv5tel-linux" "armv6l-linux" "armv7l-linux" ]; - i686 = ["i686-linux" "i686-freebsd" "i686-netbsd" "i686-cygwin"]; - mips = [ "mips64el-linux" ]; - x86_64 = ["x86_64-linux" "x86_64-darwin" "x86_64-freebsd" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin"]; + arm = filterDoubles (matchAttrs { cpu = { family = "arm"; bits = 32; }; }); + i686 = filterDoubles parse.isi686; + mips = filterDoubles (matchAttrs { cpu = { family = "mips"; }; }); + x86_64 = filterDoubles parse.isx86_64; - cygwin = ["i686-cygwin" "x86_64-cygwin"]; - darwin = ["x86_64-darwin"]; - freebsd = ["i686-freebsd" "x86_64-freebsd"]; - gnu = linux; /* ++ hurd ++ kfreebsd ++ ... */ - illumos = ["x86_64-solaris"]; - linux = ["i686-linux" "x86_64-linux" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux" "mips64el-linux"]; - netbsd = ["i686-netbsd" "x86_64-netbsd"]; - openbsd = ["i686-openbsd" "x86_64-openbsd"]; - unix = linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos; + cygwin = filterDoubles (matchAttrs { kernel = parse.kernels.cygwin; }); + darwin = filterDoubles parse.isDarwin; + freebsd = filterDoubles (matchAttrs { kernel = parse.kernels.freebsd; }); + gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }); # Should be better + illumos = filterDoubles (matchAttrs { kernel = parse.kernels.solaris; }); + linux = filterDoubles parse.isLinux; + netbsd = filterDoubles (matchAttrs { kernel = parse.kernels.netbsd; }); + openbsd = filterDoubles (matchAttrs { kernel = parse.kernels.openbsd; }); + unix = filterDoubles parse.isUnix; mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux"]; } diff --git a/lib/systems/parse.nix b/lib/systems/parse.nix index 410995f8a5d7..f6315b8ad2a9 100644 --- a/lib/systems/parse.nix +++ b/lib/systems/parse.nix @@ -6,11 +6,13 @@ with import ../types.nix; with import ../attrsets.nix; let - lib = import ./default.nix; - setTypes = type: + lib = import ../default.nix; + setTypesAssert = type: pred: mapAttrs (name: value: - setType type ({inherit name;} // value) - ); + #assert pred value; + setType type ({ inherit name; } // value)); + setTypes = type: setTypesAssert type (_: true); + in rec { @@ -22,20 +24,31 @@ rec { }; - isCpuType = x: isType "cpu-type" x - && elem x.bits [8 16 32 64 128] - && (8 < x.bits -> isSignificantByte x.significantByte); - - cpuTypes = with significantBytes; - setTypes "cpu-type" { - arm = { bits = 32; significantByte = littleEndian; }; - armv5tel = { bits = 32; significantByte = littleEndian; }; - armv7l = { bits = 32; significantByte = littleEndian; }; - i686 = { bits = 32; significantByte = littleEndian; }; - powerpc = { bits = 32; significantByte = bigEndian; }; - x86_64 = { bits = 64; significantByte = littleEndian; }; - }; + isCpuType = isType "cpu-type"; + cpuTypes = with significantBytes; setTypesAssert "cpu-type" + (x: elem x.bits [8 16 32 64 128] + && (if 8 < x.bits + then isSignificantByte x.significantByte + else !(x ? significantByte))) + { + arm = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv5tel = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv6l = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv7a = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv7l = { bits = 32; significantByte = littleEndian; family = "arm"; }; + aarch64 = { bits = 64; significantByte = littleEndian; family = "arm"; }; + i686 = { bits = 32; significantByte = littleEndian; family = "x86"; }; + x86_64 = { bits = 64; significantByte = littleEndian; family = "x86"; }; + mips64el = { bits = 32; significantByte = littleEndian; family = "mips"; }; + powerpc = { bits = 32; significantByte = bigEndian; family = "powerpc"; }; + }; + isVendor = isType "vendor"; + vendors = setTypes "vendor" { + apple = {}; + pc = {}; + unknown = {}; + }; isExecFormat = isType "exec-format"; execFormats = setTypes "exec-format" { @@ -43,84 +56,118 @@ rec { elf = {}; macho = {}; pe = {}; - unknow = {}; + unknown = {}; + }; + + isKernelFamily = isType "kernel-family"; + kernelFamilies = setTypes "kernel-family" { + bsd = {}; + unix = {}; + windows-nt = {}; + dos = {}; + }; + + isKernel = x: isType "kernel" x; + kernels = with execFormats; with kernelFamilies; setTypesAssert "kernel" + (x: isExecFormat x.execFormat && all isKernelFamily (attrValues x.families)) + { + cygwin = { execFormat = pe; families = { inherit /*unix*/ windows-nt; }; }; + darwin = { execFormat = macho; families = { inherit unix; }; }; + freebsd = { execFormat = elf; families = { inherit unix bsd; }; }; + linux = { execFormat = elf; families = { inherit unix; }; }; + netbsd = { execFormat = elf; families = { inherit unix bsd; }; }; + none = { execFormat = unknown; families = { inherit unix; }; }; + openbsd = { execFormat = elf; families = { inherit unix bsd; }; }; + solaris = { execFormat = elf; families = { inherit unix; }; }; + win32 = { execFormat = pe; families = { inherit dos; }; }; }; - isKernel = isType "kernel"; - kernels = with execFormats; - setTypes "kernel" { - cygwin = { execFormat = pe; }; - darwin = { execFormat = macho; }; - freebsd = { execFormat = elf; }; - linux = { execFormat = elf; }; - netbsd = { execFormat = elf; }; - none = { execFormat = unknow; }; - openbsd = { execFormat = elf; }; - win32 = { execFormat = pe; }; + isAbi = isType "abi"; + abis = setTypes "abi" { + gnu = {}; + msvc = {}; + eabi = {}; + androideabi = {}; + unknown = {}; + }; + + isSystem = isType "system"; + mkSystem = { cpu, vendor, kernel, abi }: + assert isCpuType cpu && isVendor vendor && isKernel kernel && isAbi abi; + setType "system" { + inherit cpu vendor kernel abi; }; - - isArchitecture = isType "architecture"; - architectures = setTypes "architecture" { - apple = {}; - pc = {}; - unknow = {}; - }; - - - isSystem = x: isType "system" x - && isCpuType x.cpu - && isArchitecture x.arch - && isKernel x.kernel; - - mkSystem = { - cpu ? cpuTypes.i686, - arch ? architectures.pc, - kernel ? kernels.linux, - name ? "${cpu.name}-${arch.name}-${kernel.name}" - }: setType "system" { - inherit name cpu arch kernel; - }; - - is64Bit = matchAttrs { cpu = { bits = 64; }; }; - isDarwin = matchAttrs { kernel = kernels.darwin; }; + is32Bit = matchAttrs { cpu = { bits = 32; }; }; isi686 = matchAttrs { cpu = cpuTypes.i686; }; - isLinux = matchAttrs { kernel = kernels.linux; }; + isx86_64 = matchAttrs { cpu = cpuTypes.x86_64; }; + isDarwin = matchAttrs { kernel = kernels.darwin; }; + isLinux = matchAttrs { kernel = kernels.linux; }; + isUnix = matchAttrs { kernel = { families = { inherit (kernelFamilies) unix; }; }; }; + isWindows = s: matchAttrs { kernel = { families = { inherit (kernelFamilies) windows-nt; }; }; } s + || matchAttrs { kernel = { families = { inherit (kernelFamilies) dos; }; }; } s; + + + mkSkeletonFromList = l: { + "2" = { cpu = elemAt l 0; kernel = elemAt l 1; }; + "4" = { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; abi = elemAt l 3; }; + "3" = # Awkwards hacks, beware! + if elemAt l 1 == "apple" + then { cpu = elemAt l 0; vendor = "apple"; kernel = elemAt l 2; } + else if (elemAt l 1 == "linux") || (elemAt l 2 == "gnu") + then { cpu = elemAt l 0; kernel = elemAt l 1; abi = elemAt l 2; } + else throw "Target specification with 3 components is ambiguous"; + }.${toString (length l)} + or (throw "system string has invalid number of hyphen-separated components"); # This should revert the job done by config.guess from the gcc compiler. - mkSystemFromString = s: let - l = lib.splitString "-" s; - + mkSystemFromSkeleton = { cpu + , # Optional, but fallback too complex for here. + # Inferred below instead. + vendor ? assert false; null + , kernel + , # Also inferred below + abi ? assert false; null + } @ args: let getCpu = name: - attrByPath [name] (throw "Unknow cpuType `${name}'.") + attrByPath [name] (throw "Unknown CPU type: ${name}") cpuTypes; - getArch = name: - attrByPath [name] (throw "Unknow architecture `${name}'.") - architectures; + getVendor = name: + attrByPath [name] (throw "Unknown vendor: ${name}") + vendors; getKernel = name: - attrByPath [name] (throw "Unknow kernel `${name}'.") + attrByPath [name] (throw "Unknown kernel: ${name}") kernels; + getAbi = name: + attrByPath [name] (throw "Unknown ABI: ${name}") + abis; + + system = rec { + cpu = getCpu args.cpu; + vendor = + /**/ if args ? vendor then getVendor args.vendor + else if isDarwin system then vendors.apple + else if isWindows system then vendors.pc + else vendors.unknown; + kernel = getKernel args.kernel; + abi = + /**/ if args ? abi then getAbi args.abi + else if isLinux system then abis.gnu + else if isWindows system then abis.gnu + else abis.unknown; + }; + + in mkSystem system; + + mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s)); + + doubleFromSystem = { cpu, vendor, kernel, abi, ... }: "${cpu.name}-${kernel.name}"; + + tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let + optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}"; + in "${cpu.name}-${vendor.name}-${kernel.name}${optAbi}"; - system = - if builtins.length l == 2 then - mkSystem rec { - name = s; - cpu = getCpu (head l); - arch = - if isDarwin system - then architectures.apple - else architectures.pc; - kernel = getKernel (head (tail l)); - } - else - mkSystem { - name = s; - cpu = getCpu (head l); - arch = getArch (head (tail l)); - kernel = getKernel (head (tail (tail l))); - }; - in assert isSystem system; system; } diff --git a/lib/tests/release.nix b/lib/tests/release.nix index f9f57424f7d0..dfa4ca2676d1 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -1,31 +1,40 @@ -{ nixpkgs }: +{ nixpkgs ? { outPath = (import ../.).cleanSource ../..; revCount = 1234; shortRev = "abcdef"; } +, # The platforms for which we build Nixpkgs. + supportedSystems ? [ builtins.currentSystem ] +, # Strip most of attributes when evaluating to spare memory usage + scrubJobs ? true +}: -with import ../.. { }; +with import ../../pkgs/top-level/release-lib.nix { inherit supportedSystems scrubJobs; }; with lib; -stdenv.mkDerivation { - name = "nixpkgs-lib-tests"; - buildInputs = [ nix ]; - NIX_PATH="nixpkgs=${nixpkgs}"; +{ + systems = import ./systems.nix { inherit lib assertTrue; }; - buildCommand = '' - datadir="${nix}/share" - export TEST_ROOT=$(pwd)/test-tmp - export NIX_BUILD_HOOK= - export NIX_CONF_DIR=$TEST_ROOT/etc - export NIX_DB_DIR=$TEST_ROOT/db - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_STORE_DIR=$TEST_ROOT/store - export PAGER=cat - cacheDir=$TEST_ROOT/binary-cache - nix-store --init + moduleSystem = pkgs.stdenv.mkDerivation { + name = "nixpkgs-lib-tests"; + buildInputs = [ pkgs.nix ]; + NIX_PATH="nixpkgs=${nixpkgs}"; - cd ${nixpkgs}/lib/tests - ./modules.sh + buildCommand = '' + datadir="${pkgs.nix}/share" + export TEST_ROOT=$(pwd)/test-tmp + export NIX_BUILD_HOOK= + export NIX_CONF_DIR=$TEST_ROOT/etc + export NIX_DB_DIR=$TEST_ROOT/db + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_STORE_DIR=$TEST_ROOT/store + export PAGER=cat + cacheDir=$TEST_ROOT/binary-cache + nix-store --init - touch $out - ''; + cd ${nixpkgs}/lib/tests + ./modules.sh + + touch $out + ''; + }; } diff --git a/lib/tests/systems.nix b/lib/tests/systems.nix new file mode 100644 index 000000000000..02e316c19fe3 --- /dev/null +++ b/lib/tests/systems.nix @@ -0,0 +1,31 @@ +# We assert that the new algorithmic way of generating these lists matches the +# way they were hard-coded before. +# +# One might think "if we exhaustively test, what's the point of procedurally +# calculating the lists anyway?". The answer is one can mindlessly update these +# tests as new platforms become supported, and then just give the diff a quick +# sanity check before committing :). +{ lib, assertTrue }: + +with lib.systems.doubles; + +let mseteq = x: y: lib.sort lib.lessThan x == lib.sort lib.lessThan y; in + +{ + all = assertTrue (mseteq all (linux ++ darwin ++ cygwin ++ freebsd ++ openbsd ++ netbsd ++ illumos)); + + arm = assertTrue (mseteq arm [ "armv5tel-linux" "armv6l-linux" "armv7l-linux" ]); + i686 = assertTrue (mseteq i686 [ "i686-linux" "i686-freebsd" "i686-netbsd" "i686-cygwin" ]); + mips = assertTrue (mseteq mips [ "mips64el-linux" ]); + x86_64 = assertTrue (mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" ]); + + cygwin = assertTrue (mseteq cygwin [ "i686-cygwin" "x86_64-cygwin" ]); + darwin = assertTrue (mseteq darwin [ "x86_64-darwin" ]); + freebsd = assertTrue (mseteq freebsd [ "i686-freebsd" "x86_64-freebsd" ]); + gnu = assertTrue (mseteq gnu (linux /* ++ hurd ++ kfreebsd ++ ... */)); + illumos = assertTrue (mseteq illumos [ "x86_64-solaris" ]); + linux = assertTrue (mseteq linux [ "i686-linux" "x86_64-linux" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "aarch64-linux" "mips64el-linux" ]); + netbsd = assertTrue (mseteq netbsd [ "i686-netbsd" "x86_64-netbsd" ]); + openbsd = assertTrue (mseteq openbsd [ "i686-openbsd" "x86_64-openbsd" ]); + unix = assertTrue (mseteq unix (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos)); +} diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index c68c7d3721e2..67f3d55fa167 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -47,11 +47,11 @@ in # cause false negatives. testEqualOne = path: system: let f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs)); - in assert + in assertTrue ( f path { inherit system; } == f (["buildPackages"] ++ path) { inherit system crossSystem; }; - true; + ); testEqual = path: systems: forAllSupportedSystems systems (testEqualOne path); diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index a5fdb9522060..edb760939cb3 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -46,6 +46,11 @@ rec { pkgs_x86_64_cygwin = allPackages { system = "x86_64-cygwin"; }; + assertTrue = bool: + if bool + then pkgs.runCommand "evaluated-to-true" {} "touch $out" + else pkgs.runCommand "evaluated-to-false" {} "false"; + /* The working or failing mails for cross builds will be sent only to the following maintainers, as most package maintainers will not be interested in the result of cross building a package. */ diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index f2feb29f0cd7..2eae99eaef11 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -26,7 +26,7 @@ let metrics = import ./metrics.nix { inherit pkgs nixpkgs; }; manual = import ../../doc; - lib-tests = import ../../lib/tests/release.nix { inherit nixpkgs; }; + lib-tests = import ../../lib/tests/release.nix { inherit nixpkgs supportedSystems scrubJobs; }; unstable = pkgs.releaseTools.aggregate { name = "nixpkgs-${jobs.tarball.version}"; @@ -35,7 +35,6 @@ let [ jobs.tarball jobs.metrics jobs.manual - jobs.lib-tests jobs.stdenv.x86_64-linux jobs.stdenv.i686-linux jobs.stdenv.x86_64-darwin @@ -62,7 +61,8 @@ let jobs.git.x86_64-darwin jobs.mysql.x86_64-darwin jobs.vim.x86_64-darwin - ] ++ lib.collect lib.isDerivation jobs.stdenvBootstrapTools; + ] ++ lib.collect lib.isDerivation jobs.stdenvBootstrapTools + ++ lib.collect lib.isDerivation jobs.lib-tests; }; } // (lib.optionalAttrs (builtins.elem "i686-linux" supportedSystems) { stdenvBootstrapTools.i686-linux =