llvm: 3.7.1 -> 3.8.0 (close #13801)

vcunat's review:
 - let's not switch the default versions of llvm* for now
 - the only changes I see is adding python to clang's buildInputs
   and using the big so-file as discussed in #12759
   (BUILD_SHARED_LIBS -> LLVM_LINK_LLVM_DYLIB)
 - in future it will be nice to split libLLVM into a separate output

pkgs/development/compilers/llvm/3.8/clang/default.nix

@@ -0,0 +1,55 @@
+{ stdenv, fetch, cmake, libxml2, libedit, llvm, version, clang-tools-extra_src, python }:
+
+let
+  gcc = if stdenv.cc.isGNU then stdenv.cc.cc else stdenv.cc.cc.gcc;
+in stdenv.mkDerivation {
+  name = "clang-${version}";
+
+  unpackPhase = ''
+    unpackFile ${fetch "cfe" "1ybcac8hlr9vl3wg8s4v6cp0c0qgqnwprsv85lihbkq3vqv94504"}
+    mv cfe-${version}.src clang
+    sourceRoot=$PWD/clang
+    unpackFile ${clang-tools-extra_src}
+    mv clang-tools-extra-* $sourceRoot/tools/extra
+  '';
+
+  buildInputs = [ cmake libedit libxml2 llvm python ];
+
+  cmakeFlags = [
+    "-DCMAKE_BUILD_TYPE=Release"
+    "-DCMAKE_CXX_FLAGS=-std=c++11"
+  ] ++
+  # Maybe with compiler-rt this won't be needed?
+  (stdenv.lib.optional stdenv.isLinux "-DGCC_INSTALL_PREFIX=${gcc}") ++
+  (stdenv.lib.optional (stdenv.cc.libc != null) "-DC_INCLUDE_DIRS=${stdenv.cc.libc}/include");
+
+  patches = [ ./purity.patch ];
+
+  postPatch = ''
+    sed -i -e 's/Args.hasArg(options::OPT_nostdlibinc)/true/' lib/Driver/Tools.cpp
+    sed -i -e 's/DriverArgs.hasArg(options::OPT_nostdlibinc)/true/' lib/Driver/ToolChains.cpp
+  '';
+
+  # Clang expects to find LLVMgold in its own prefix
+  # Clang expects to find sanitizer libraries in its own prefix
+  postInstall = ''
+    ln -sv ${llvm}/lib/LLVMgold.so $out/lib
+    ln -sv ${llvm}/lib/clang/${version}/lib $out/lib/clang/${version}/
+    ln -sv $out/bin/clang $out/bin/cpp
+  '';
+
+  enableParallelBuilding = true;
+
+  passthru = {
+    isClang = true;
+  } // stdenv.lib.optionalAttrs stdenv.isLinux {
+    inherit gcc;
+  };
+
+  meta = {
+    description = "A c, c++, objective-c, and objective-c++ frontend for the llvm compiler";
+    homepage    = http://llvm.org/;
+    license     = stdenv.lib.licenses.bsd3;
+    platforms   = stdenv.lib.platforms.all;
+  };
+}

pkgs/development/compilers/llvm/3.8/clang/purity.patch

@@ -0,0 +1,17 @@
+--- a/lib/Driver/Tools.cpp	2016-02-12 15:51:41.000000000 -0700
++++ b/lib/Driver/Tools.cpp	2016-03-08 15:39:06.790111122 -0700
+@@ -8833,15 +8833,6 @@
+     CmdArgs.push_back("-shared");
+   }
+ 
+-  if (Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb ||
+-      Arch == llvm::Triple::thumb || Arch == llvm::Triple::thumbeb ||
+-      (!Args.hasArg(options::OPT_static) &&
+-       !Args.hasArg(options::OPT_shared))) {
+-    CmdArgs.push_back("-dynamic-linker");
+-    CmdArgs.push_back(Args.MakeArgString(
+-        D.DyldPrefix + getLinuxDynamicLinker(Args, ToolChain)));
+-  }
+-
+   CmdArgs.push_back("-o");
+   CmdArgs.push_back(Output.getFilename());

pkgs/development/compilers/llvm/3.8/default.nix

@@ -0,0 +1,35 @@
+{ newScope, stdenv, isl, fetchurl, overrideCC, wrapCC }:
+let
+  callPackage = newScope (self // { inherit stdenv isl version fetch; });
+
+  version = "3.8.0";
+
+  fetch = fetch_v version;
+  fetch_v = ver: name: sha256: fetchurl {
+    url = "http://llvm.org/releases/${ver}/${name}-${ver}.src.tar.xz";
+    inherit sha256;
+  };
+
+  compiler-rt_src = fetch "compiler-rt" "1c2nkp9563873ffz22qmhc0wakgj428pch8rmhym8agjamz3ily8";
+  clang-tools-extra_src = fetch "clang-tools-extra" "1i0yrgj8qrzjjswraz0i55lg92ljpqhvjr619d268vka208aigdg";
+
+  self = {
+    llvm = callPackage ./llvm.nix {
+      inherit compiler-rt_src stdenv;
+    };
+
+    clang-unwrapped = callPackage ./clang {
+      inherit clang-tools-extra_src stdenv;
+    };
+
+    clang = wrapCC self.clang-unwrapped;
+
+    stdenv = overrideCC stdenv self.clang;
+
+    lldb = callPackage ./lldb.nix {};
+
+    libcxx = callPackage ./libc++ {};
+
+    libcxxabi = callPackage ./libc++abi.nix {};
+  };
+in self

pkgs/development/compilers/llvm/3.8/libc++/darwin.patch

@@ -0,0 +1,30 @@
+diff -ru -x '*~' libcxx-3.4.2.src-orig/lib/CMakeLists.txt libcxx-3.4.2.src/lib/CMakeLists.txt
+--- libcxx-3.4.2.src-orig/lib/CMakeLists.txt	2013-11-15 18:18:57.000000000 +0100
++++ libcxx-3.4.2.src/lib/CMakeLists.txt	2014-09-24 14:04:01.000000000 +0200
+@@ -56,7 +56,7 @@
+       "-compatibility_version 1"
+       "-current_version ${LIBCXX_VERSION}"
+       "-install_name /usr/lib/libc++.1.dylib"
+-      "-Wl,-reexport_library,/usr/lib/libc++abi.dylib"
++      "-Wl,-reexport_library,${LIBCXX_LIBCXXABI_LIB_PATH}/libc++abi.dylib"
+       "-Wl,-unexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/libc++unexp.exp"
+       "/usr/lib/libSystem.B.dylib")
+   else()
+@@ -64,14 +64,14 @@
+       list(FIND ${CMAKE_OSX_ARCHITECTURES} "armv7" OSX_HAS_ARMV7)
+       if (OSX_HAS_ARMV7)
+         set(OSX_RE_EXPORT_LINE
+-          "${CMAKE_OSX_SYSROOT}/usr/lib/libc++abi.dylib"
++          "${CMAKE_OSX_SYSROOT}${LIBCXX_LIBCXXABI_LIB_PATH}/libc++abi.dylib"
+           "-Wl,-reexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/libc++sjlj-abi.exp")
+       else()
+         set(OSX_RE_EXPORT_LINE
+-          "-Wl,-reexport_library,${CMAKE_OSX_SYSROOT}/usr/lib/libc++abi.dylib")
++          "-Wl,-reexport_library,${CMAKE_OSX_SYSROOT}${LIBCXX_LIBCXXABI_LIB_PATH}/libc++abi.dylib")
+       endif()
+     else()
+-      set (OSX_RE_EXPORT_LINE "/usr/lib/libc++abi.dylib -Wl,-reexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/libc++abi${LIBCXX_LIBCPPABI_VERSION}.exp")
++      set (OSX_RE_EXPORT_LINE "${LIBCXX_LIBCXXABI_LIB_PATH}/libc++abi.dylib -Wl,-reexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/libc++abi${LIBCXX_LIBCPPABI_VERSION}.exp")
+     endif()
+ 
+     list(APPEND link_flags

pkgs/development/compilers/llvm/3.8/libc++/default.nix

@@ -0,0 +1,40 @@
+{ lib, stdenv, fetch, cmake, libcxxabi, fixDarwinDylibNames, version }:
+
+stdenv.mkDerivation rec {
+  name = "libc++-${version}";
+
+  src = fetch "libcxx" "0i7iyzk024krda5spfpfi8jksh83yp3bxqkal0xp76ffi11bszrm";
+
+  postUnpack = ''
+    unpackFile ${libcxxabi.src}
+  '';
+
+  preConfigure = ''
+    # Get headers from the cxxabi source so we can see private headers not installed by the cxxabi package
+    cmakeFlagsArray=($cmakeFlagsArray -DLIBCXX_CXX_ABI_INCLUDE_PATHS="$NIX_BUILD_TOP/libcxxabi-${version}.src/include")
+  '';
+
+  patches = [ ./darwin.patch ];
+
+  buildInputs = [ cmake libcxxabi ] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
+
+  cmakeFlags =
+    [ "-DCMAKE_BUILD_TYPE=Release"
+      "-DLIBCXX_LIBCXXABI_LIB_PATH=${libcxxabi}/lib"
+      "-DLIBCXX_LIBCPPABI_VERSION=2"
+      "-DLIBCXX_CXX_ABI=libcxxabi"
+    ];
+
+  enableParallelBuilding = true;
+
+  linkCxxAbi = stdenv.isLinux;
+
+  setupHook = ./setup-hook.sh;
+
+  meta = {
+    homepage = http://libcxx.llvm.org/;
+    description = "A new implementation of the C++ standard library, targeting C++11";
+    license = "BSD";
+    platforms = stdenv.lib.platforms.unix;
+  };
+}

pkgs/development/compilers/llvm/3.8/libc++/setup-hook.sh

@@ -0,0 +1,3 @@
+linkCxxAbi="@linkCxxAbi@"
+export NIX_CXXSTDLIB_COMPILE+=" -isystem @out@/include/c++/v1"
+export NIX_CXXSTDLIB_LINK=" -stdlib=libc++${linkCxxAbi:+" -lc++abi"}"

pkgs/development/compilers/llvm/3.8/libc++abi.nix

@@ -0,0 +1,47 @@
+{ stdenv, cmake, fetch, libcxx, libunwind, llvm, version }:
+
+stdenv.mkDerivation {
+  name = "libc++abi-${version}";
+
+  src = fetch "libcxxabi" "0ambfcmr2nh88hx000xb7yjm9lsqjjz49w5mlf6dlxzmj3nslzx4";
+
+  buildInputs = [ cmake ] ++ stdenv.lib.optional (!stdenv.isDarwin && !stdenv.isFreeBSD) libunwind;
+
+  postUnpack = ''
+    unpackFile ${libcxx.src}
+    unpackFile ${llvm.src}
+    export NIX_CFLAGS_COMPILE+=" -I$PWD/include"
+    export cmakeFlags="-DLLVM_PATH=$PWD/$(ls -d llvm-*) -DLIBCXXABI_LIBCXX_INCLUDES=$PWD/$(ls -d libcxx-*)/include"
+  '' + stdenv.lib.optionalString stdenv.isDarwin ''
+    export TRIPLE=x86_64-apple-darwin
+  '';
+
+  installPhase = if stdenv.isDarwin
+    then ''
+      for file in lib/*.dylib; do
+        # this should be done in CMake, but having trouble figuring out
+        # the magic combination of necessary CMake variables
+        # if you fancy a try, take a look at
+        # http://www.cmake.org/Wiki/CMake_RPATH_handling
+        install_name_tool -id $out/$file $file
+      done
+      make install
+      install -d 755 $out/include
+      install -m 644 ../include/*.h $out/include
+    ''
+    else ''
+      install -d -m 755 $out/include $out/lib
+      install -m 644 lib/libc++abi.so.1.0 $out/lib
+      install -m 644 ../include/cxxabi.h $out/include
+      ln -s libc++abi.so.1.0 $out/lib/libc++abi.so
+      ln -s libc++abi.so.1.0 $out/lib/libc++abi.so.1
+    '';
+
+  meta = {
+    homepage = http://libcxxabi.llvm.org/;
+    description = "A new implementation of low level support for a standard C++ library";
+    license = "BSD";
+    maintainers = with stdenv.lib.maintainers; [ vlstill ];
+    platforms = stdenv.lib.platforms.unix;
+  };
+}

pkgs/development/compilers/llvm/3.8/lldb.nix

@@ -0,0 +1,49 @@
+{ stdenv
+, fetch
+, cmake
+, zlib
+, ncurses
+, swig
+, which
+, libedit
+, llvm
+, clang-unwrapped
+, python
+, version
+}:
+
+stdenv.mkDerivation {
+  name = "lldb-${version}";
+
+  src = fetch "lldb" "008fdbyza13ym3v0xpans4z4azw4y16hcbgrrnc4rx2mxwaw62ws";
+
+  patchPhase = ''
+    sed -i 's|/usr/bin/env||' \
+      scripts/Python/finish-swig-Python-LLDB.sh \
+      scripts/Python/build-swig-Python.sh
+  '';
+
+  buildInputs = [ cmake python which swig ncurses zlib libedit ];
+
+  preConfigure = ''
+    export CXXFLAGS="-pthread"
+    export LDFLAGS="-ldl"
+  '';
+
+  cmakeFlags = [
+    "-DCMAKE_BUILD_TYPE=Release"
+    "-DLLDB_PATH_TO_LLVM_BUILD=${llvm}"
+    "-DLLDB_PATH_TO_CLANG_BUILD=${clang-unwrapped}"
+    "-DPYTHON_VERSION_MAJOR=2"
+    "-DPYTHON_VERSION_MINOR=7"
+  ];
+
+  enableParallelBuilding = true;
+
+  meta = {
+    description = "A next-generation high-performance debugger";
+    homepage    = http://llvm.org/;
+    license     = stdenv.lib.licenses.bsd3;
+    platforms   = stdenv.lib.platforms.all;
+  };
+}

pkgs/development/compilers/llvm/3.8/llvm.nix

@@ -0,0 +1,80 @@
+{ stdenv
+, fetch
+, perl
+, groff
+, cmake
+, python
+, libffi
+, binutils
+, libxml2
+, valgrind
+, ncurses
+, version
+, zlib
+, compiler-rt_src
+, libcxxabi
+, debugVersion ? false
+, enableSharedLibraries ? !stdenv.isDarwin
+}:
+
+let
+  src = fetch "llvm" "0ikfq0gxac8xpvxj23l4hk8f12ydx48fljgrz1gl9xp0ks704nsm";
+in stdenv.mkDerivation rec {
+  name = "llvm-${version}";
+
+  unpackPhase = ''
+    unpackFile ${src}
+    mv llvm-${version}.src llvm
+    sourceRoot=$PWD/llvm
+    unpackFile ${compiler-rt_src}
+    mv compiler-rt-* $sourceRoot/projects/compiler-rt
+  '';
+
+  buildInputs = [ perl groff cmake libxml2 python libffi ]
+    ++ stdenv.lib.optional stdenv.isDarwin libcxxabi;
+
+  propagatedBuildInputs = [ ncurses zlib ];
+
+  # hacky fix: created binaries need to be run before installation
+  preBuild = ''
+    mkdir -p $out/
+    ln -sv $PWD/lib $out
+  '';
+
+  cmakeFlags = with stdenv; [
+    "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}"
+    "-DLLVM_INSTALL_UTILS=ON"  # Needed by rustc
+    "-DLLVM_BUILD_TESTS=ON"
+    "-DLLVM_ENABLE_FFI=ON"
+    "-DLLVM_ENABLE_RTTI=ON"
+  ] ++ stdenv.lib.optional enableSharedLibraries [
+    "-DLLVM_LINK_LLVM_DYLIB=ON"
+  ] ++ stdenv.lib.optional (!isDarwin)
+    "-DLLVM_BINUTILS_INCDIR=${binutils}/include"
+    ++ stdenv.lib.optionals ( isDarwin) [
+    "-DLLVM_ENABLE_LIBCXX=ON"
+    "-DCAN_TARGET_i386=false"
+  ];
+
+  postBuild = ''
+    rm -fR $out
+
+    paxmark m bin/{lli,llvm-rtdyld}
+
+    paxmark m unittests/ExecutionEngine/JIT/JITTests
+    paxmark m unittests/ExecutionEngine/MCJIT/MCJITTests
+    paxmark m unittests/Support/SupportTests
+  '';
+
+  enableParallelBuilding = true;
+
+  passthru.src = src;
+
+  meta = {
+    description = "Collection of modular and reusable compiler and toolchain technologies";
+    homepage    = http://llvm.org/;
+    license     = stdenv.lib.licenses.bsd3;
+    maintainers = with stdenv.lib.maintainers; [ lovek323 raskin viric ];
+    platforms   = stdenv.lib.platforms.all;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -3921,6 +3921,7 @@ in
 
   clang = llvmPackages.clang;
 
+  clang_38 = llvmPackages_38.clang;
   clang_37 = llvmPackages_37.clang;
   clang_36 = llvmPackages_36.clang;
   clang_35 = wrapCC llvmPackages_35.clang;
@@ -4418,6 +4419,7 @@ in
 
   llvm = llvmPackages.llvm;
 
+  llvm_38 = llvmPackages_38.llvm;
   llvm_37 = llvmPackages_37.llvm;
   llvm_36 = llvmPackages_36.llvm;
   llvm_35 = llvmPackages_35.llvm;
@@ -4445,6 +4447,10 @@ in
     inherit (stdenvAdapters) overrideCC;
   };
 
+  llvmPackages_38 = callPackage ../development/compilers/llvm/3.8 {
+    inherit (stdenvAdapters) overrideCC;
+  };
+
   manticore = callPackage ../development/compilers/manticore { };
 
   mentorToolchains = recurseIntoAttrs (