From d7c1d04af4085e0ae6fecbffdb0a19299de598e8 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Fri, 17 Aug 2018 20:48:40 -0400 Subject: [PATCH 1/3] mkl: init at 2019.0.117 This packags the Intel Math Kernel library on x86-64 platforms, which is a dependency for many data science and machine learning packages. Upstream, Intel provides proprietary binary RPMs with a permissive redistribution license. These have been repackaged in both Debian and Anaconda, so we are not the first distribution to redistribute. --- lib/licenses.nix | 8 +++ .../libraries/science/math/mkl/default.nix | 55 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 4 ++ 3 files changed, 67 insertions(+) create mode 100644 pkgs/development/libraries/science/math/mkl/default.nix diff --git a/lib/licenses.nix b/lib/licenses.nix index c4db280645a4..64d538df3c90 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -387,6 +387,14 @@ lib.mapAttrs (n: v: v // { shortName = n; }) rec { fullName = "ISC License"; }; + # Proprietary binaries; free to redistribute without modification. + issl = { + fullName = "Intel Simplified Software License"; + url = https://software.intel.com/en-us/license/intel-simplified-software-license; + free = false; + }; + + lgpl2 = spdx { spdxId = "LGPL-2.0"; fullName = "GNU Library General Public License v2 only"; diff --git a/pkgs/development/libraries/science/math/mkl/default.nix b/pkgs/development/libraries/science/math/mkl/default.nix new file mode 100644 index 000000000000..7412ecdba21d --- /dev/null +++ b/pkgs/development/libraries/science/math/mkl/default.nix @@ -0,0 +1,55 @@ +{ stdenvNoCC, writeText, fetchurl, rpm, cpio, openmp }: + +stdenvNoCC.mkDerivation rec { + name = "mkl-${version}"; + version = "${date}.${rel}"; + date = "2019.0"; + rel = "117"; + + src = fetchurl { + url = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/13575/l_mkl_${version}.tgz"; + sha256 = "1bf7i54iqlf7x7fn8kqwmi06g30sxr6nq3ac0r871i6g0p3y47sf"; + }; + + buildInputs = [ rpm cpio ]; + propagatedBuildInputs = [ openmp ]; + + buildPhase = '' + rpm2cpio rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm | cpio -idmv + rpm2cpio rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm | cpio -idmv + ''; + + installPhase = '' + mkdir -p $out/lib + cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/include $out/ + cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/lib/intel64_lin/* $out/lib/ + cp license.txt $out/lib/ + ''; + + # Per license agreement, do not modify the binary + dontStrip = true; + dontPatchELF = true; + + # Some mkl calls require openmp, but Intel does not add these to SO_NEEDED and + # instructs users to put openmp on their LD_LIBRARY_PATH. + setupHook = writeText "setup-hook.sh" '' + addOpenmp() { + addToSearchPath LD_LIBRARY_PATH ${openmp}/lib + } + addEnvHooks "$targetOffset" addOpenmp + ''; + + meta = with stdenvNoCC.lib; { + description = "Intel Math Kernel Library"; + longDescription = '' + Intel Math Kernel Library (Intel MKL) optimizes code with minimal effort + for future generations of Intel processors. It is compatible with your + choice of compilers, languages, operating systems, and linking and + threading models. + ''; + homepage = https://software.intel.com/en-us/mkl; + license = licenses.issl; + platforms = [ "x86_64-linux" ]; + maintainers = [ maintainers.bhipple ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 18ad8892e55f..f9eb92364af2 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -20858,6 +20858,10 @@ with pkgs; m4rie = callPackage ../development/libraries/science/math/m4rie { }; + mkl = callPackage ../development/libraries/science/math/mkl { + inherit (llvmPackages) openmp; + }; + nasc = callPackage ../applications/science/math/nasc { }; openblas = callPackage ../development/libraries/science/math/openblas { }; From ba02a2c3715e03329c143723c8c2c6b43fafd3a6 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Tue, 2 Oct 2018 22:53:48 -0400 Subject: [PATCH 2/3] Add Darwin as a supported platform --- .../libraries/science/math/mkl/default.nix | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/pkgs/development/libraries/science/math/mkl/default.nix b/pkgs/development/libraries/science/math/mkl/default.nix index 7412ecdba21d..930dcd87a635 100644 --- a/pkgs/development/libraries/science/math/mkl/default.nix +++ b/pkgs/development/libraries/science/math/mkl/default.nix @@ -1,4 +1,4 @@ -{ stdenvNoCC, writeText, fetchurl, rpm, cpio, openmp }: +{ stdenvNoCC, writeText, fetchurl, rpmextract, openmp, undmg }: stdenvNoCC.mkDerivation rec { name = "mkl-${version}"; @@ -6,24 +6,40 @@ stdenvNoCC.mkDerivation rec { date = "2019.0"; rel = "117"; - src = fetchurl { - url = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/13575/l_mkl_${version}.tgz"; - sha256 = "1bf7i54iqlf7x7fn8kqwmi06g30sxr6nq3ac0r871i6g0p3y47sf"; - }; + src = if stdenvNoCC.isDarwin + then + (fetchurl { + url = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/13565/m_mkl_${version}.dmg"; + sha256 = "1f1jppac7vqwn00hkws0p4njx38ajh0n25bsjyb5d7jcacwfvm02"; + }) + else + (fetchurl { + url = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/13575/l_mkl_${version}.tgz"; + sha256 = "1bf7i54iqlf7x7fn8kqwmi06g30sxr6nq3ac0r871i6g0p3y47sf"; + }); - buildInputs = [ rpm cpio ]; + buildInputs = if stdenvNoCC.isDarwin then [ undmg ] else [ rpmextract ]; propagatedBuildInputs = [ openmp ]; - buildPhase = '' - rpm2cpio rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm | cpio -idmv - rpm2cpio rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm | cpio -idmv - ''; + buildPhase = if stdenvNoCC.isDarwin then '' + for f in Contents/Resources/pkg/*.tgz; do + tar xzvf $f + done + '' else '' + rpmextract rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm + rpmextract rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm + '' ; - installPhase = '' - mkdir -p $out/lib - cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/include $out/ - cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/lib/intel64_lin/* $out/lib/ - cp license.txt $out/lib/ + installPhase = if stdenvNoCC.isDarwin then '' + mkdir -p $out/lib + cp -r compilers_and_libraries_${version}/mac/mkl/include $out/ + cp -r compilers_and_libraries_${version}/mac/mkl/lib/* $out/lib/ + cp -r compilers_and_libraries_${version}/licensing/mkl/en/license.txt $out/lib/ + '' else '' + mkdir -p $out/lib + cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/include $out/ + cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/lib/intel64_lin/* $out/lib/ + cp license.txt $out/lib/ ''; # Per license agreement, do not modify the binary @@ -48,8 +64,8 @@ stdenvNoCC.mkDerivation rec { threading models. ''; homepage = https://software.intel.com/en-us/mkl; - license = licenses.issl; - platforms = [ "x86_64-linux" ]; + license = [ licenses.issl licenses.unfreeRedistributable ]; + platforms = [ "x86_64-linux" "x86_64-darwin" ]; maintainers = [ maintainers.bhipple ]; }; } From 0b82df8dbe8540eb8a3acc7ff093ea872ae131a8 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Thu, 4 Oct 2018 02:24:21 +0000 Subject: [PATCH 3/3] Convert entire mkl package to a fixed-output derivation --- .../libraries/science/math/mkl/default.nix | 36 +++++++++++++------ pkgs/top-level/all-packages.nix | 4 +-- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pkgs/development/libraries/science/math/mkl/default.nix b/pkgs/development/libraries/science/math/mkl/default.nix index 930dcd87a635..37814047f975 100644 --- a/pkgs/development/libraries/science/math/mkl/default.nix +++ b/pkgs/development/libraries/science/math/mkl/default.nix @@ -1,4 +1,20 @@ -{ stdenvNoCC, writeText, fetchurl, rpmextract, openmp, undmg }: +{ stdenvNoCC, writeText, fetchurl, rpmextract, undmg }: +/* + Some (but not all) mkl functions require openmp, but Intel does not add these + to SO_NEEDED and instructs users to put openmp on their LD_LIBRARY_PATH. If + you are using mkl and your library/application is using some of the functions + that require openmp, add a setupHook like this to your package: + + setupHook = writeText "setup-hook.sh" '' + addOpenmp() { + addToSearchPath LD_LIBRARY_PATH ${openmp}/lib + } + addEnvHooks "$targetOffset" addOpenmp + ''; + + We do not add the setup hook here, because avoiding it allows this large + package to be a fixed-output derivation with better cache efficiency. + */ stdenvNoCC.mkDerivation rec { name = "mkl-${version}"; @@ -19,7 +35,6 @@ stdenvNoCC.mkDerivation rec { }); buildInputs = if stdenvNoCC.isDarwin then [ undmg ] else [ rpmextract ]; - propagatedBuildInputs = [ openmp ]; buildPhase = if stdenvNoCC.isDarwin then '' for f in Contents/Resources/pkg/*.tgz; do @@ -28,7 +43,7 @@ stdenvNoCC.mkDerivation rec { '' else '' rpmextract rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm rpmextract rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm - '' ; + ''; installPhase = if stdenvNoCC.isDarwin then '' mkdir -p $out/lib @@ -46,14 +61,13 @@ stdenvNoCC.mkDerivation rec { dontStrip = true; dontPatchELF = true; - # Some mkl calls require openmp, but Intel does not add these to SO_NEEDED and - # instructs users to put openmp on their LD_LIBRARY_PATH. - setupHook = writeText "setup-hook.sh" '' - addOpenmp() { - addToSearchPath LD_LIBRARY_PATH ${openmp}/lib - } - addEnvHooks "$targetOffset" addOpenmp - ''; + # Since these are unmodified binaries from Intel, they do not depend on stdenv + # and we can make them fixed-output derivations for cache efficiency. + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = if stdenvNoCC.isDarwin + then "1224dln7n8px1rk8biiggf77wjhxh8mzw0hd8zlyjm8i6j8w7i12" + else "0d8ai0wi8drp071acqkm1wv6vyg12010y843y56zzi1pql81xqvx"; meta = with stdenvNoCC.lib; { description = "Intel Math Kernel Library"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f9eb92364af2..cba1fa778d68 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -20858,9 +20858,7 @@ with pkgs; m4rie = callPackage ../development/libraries/science/math/m4rie { }; - mkl = callPackage ../development/libraries/science/math/mkl { - inherit (llvmPackages) openmp; - }; + mkl = callPackage ../development/libraries/science/math/mkl { }; nasc = callPackage ../applications/science/math/nasc { };