WIP: buildPythonPackages now uses wheels internally

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -0,0 +1,47 @@
+{ stdenv, python, fetchurl, makeWrapper, unzip }:
+
+let
+  wheel_source = fetchurl {
+    url = "https://pypi.python.org/packages/py2.py3/w/wheel/wheel-0.26.0-py2.py3-none-any.whl";
+    sha256 = "1sl642ncvipqx0hzypvl5hsiqngy0sib0kq242g4mic7vnid6bn9";
+  };
+  setuptools_source = fetchurl {
+    url = "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-18.2-py2.py3-none-any.whl";
+    sha256 = "0jhafl8wmjc8xigl1ib5hqiq9crmipcz0zcga52riymgqbf2bzh4";
+  };
+in stdenv.mkDerivation rec {
+  name = "python-${python.version}-bootstrapped-pip-${version}";
+  version = "7.1.2";
+
+  src = fetchurl {
+    url = "https://pypi.python.org/packages/py2.py3/p/pip/pip-${version}-py2.py3-none-any.whl";
+    sha256 = "133hx6jaspm6hd02gza66lng37l65yficc2y2x1gh16fbhxrilxr";
+  };
+
+  unpackPhase = ''
+    mkdir -p $out/${python.sitePackages}
+    unzip -d $out/${python.sitePackages} $src
+    unzip -d $out/${python.sitePackages} ${setuptools_source}
+    unzip -d $out/${python.sitePackages} ${wheel_source}
+  '';
+
+  buildInputs = [ python makeWrapper unzip ];
+
+  installPhase = ''
+    mkdir -p $out/bin
+
+    # patch pip to support "pip install --prefix"
+    pushd $out/${python.sitePackages}/
+    patch -p1 < ${./pip-7.0.1-prefix.patch}
+    popd
+
+    # install pip binary
+    echo '${python.interpreter} -m pip "$@"' > $out/bin/pip
+    chmod +x $out/bin/pip
+
+    # wrap binaries with PYTHONPATH
+    for f in $out/bin/*; do
+      wrapProgram $f --prefix PYTHONPATH ":" $out/${python.sitePackages}/
+    done
+  '';
+}

pkgs/development/python-modules/bootstrapped-pip/pip-7.0.1-prefix.patch

@@ -0,0 +1,119 @@
+commit e87c83d95bb91acdca92202e94488ca51a70e059
+Author: Domen Kožar <domen@dev.si>
+Date:   Mon Nov 16 17:39:44 2015 +0100
+
+    WIP
+
+diff --git a/pip/commands/install.py b/pip/commands/install.py
+index dbcf100..05d5a08 100644
+--- a/pip/commands/install.py
++++ b/pip/commands/install.py
+@@ -139,6 +139,13 @@ class InstallCommand(RequirementCommand):
+                  "directory.")
+ 
+         cmd_opts.add_option(
++            '--prefix',
++            dest='prefix_path',
++            metavar='dir',
++            default=None,
++            help="Installation prefix where lib, bin and other top-level folders are placed")
++
++        cmd_opts.add_option(
+             "--compile",
+             action="store_true",
+             dest="compile",
+@@ -309,6 +316,7 @@ class InstallCommand(RequirementCommand):
+                             install_options,
+                             global_options,
+                             root=options.root_path,
++                            prefix=options.prefix_path,
+                         )
+                         reqs = sorted(
+                             requirement_set.successfully_installed,
+diff --git a/pip/locations.py b/pip/locations.py
+index 4e6f65d..43aeb1f 100644
+--- a/pip/locations.py
++++ b/pip/locations.py
+@@ -163,7 +163,7 @@ site_config_files = [
+ 
+ 
+ def distutils_scheme(dist_name, user=False, home=None, root=None,
+-                     isolated=False):
++                     isolated=False, prefix=None):
+     """
+     Return a distutils install scheme
+     """
+@@ -187,6 +187,8 @@ def distutils_scheme(dist_name, user=False, home=None, root=None,
+     i.user = user or i.user
+     if user:
+         i.prefix = ""
++    else:
++        i.prefix = prefix or i.prefix
+     i.home = home or i.home
+     i.root = root or i.root
+     i.finalize_options()
+diff --git a/pip/req/req_install.py b/pip/req/req_install.py
+index 7c5bf8f..6f80a18 100644
+--- a/pip/req/req_install.py
++++ b/pip/req/req_install.py
+@@ -792,7 +792,7 @@ exec(compile(
+         else:
+             return True
+ 
+-    def install(self, install_options, global_options=[], root=None):
++    def install(self, install_options, global_options=[], root=None, prefix=None):
+         if self.editable:
+             self.install_editable(install_options, global_options)
+             return
+@@ -800,7 +800,7 @@ exec(compile(
+             version = pip.wheel.wheel_version(self.source_dir)
+             pip.wheel.check_compatibility(version, self.name)
+ 
+-            self.move_wheel_files(self.source_dir, root=root)
++            self.move_wheel_files(self.source_dir, root=root, prefix=prefix)
+             self.install_succeeded = True
+             return
+ 
+@@ -833,6 +833,8 @@ exec(compile(
+ 
+             if root is not None:
+                 install_args += ['--root', root]
++            if prefix is not None:
++                install_args += ['--prefix', prefix]
+ 
+             if self.pycompile:
+                 install_args += ["--compile"]
+@@ -988,12 +990,13 @@ exec(compile(
+     def is_wheel(self):
+         return self.link and self.link.is_wheel
+ 
+-    def move_wheel_files(self, wheeldir, root=None):
++    def move_wheel_files(self, wheeldir, root=None, prefix=None):
+         move_wheel_files(
+             self.name, self.req, wheeldir,
+             user=self.use_user_site,
+             home=self.target_dir,
+             root=root,
++            prefix=prefix,
+             pycompile=self.pycompile,
+             isolated=self.isolated,
+         )
+diff --git a/pip/wheel.py b/pip/wheel.py
+index 403f48b..14eb141 100644
+--- a/pip/wheel.py
++++ b/pip/wheel.py
+@@ -234,12 +234,12 @@ def get_entrypoints(filename):
+ 
+ 
+ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None,
+-                     pycompile=True, scheme=None, isolated=False):
++                     pycompile=True, scheme=None, isolated=False, prefix=None):
+     """Install a wheel"""
+ 
+     if not scheme:
+         scheme = distutils_scheme(
+-            name, user=user, home=home, root=root, isolated=isolated
++            name, user=user, home=home, root=root, isolated=isolated, prefix=prefix,
+         )
+ 
+     if root_is_purelib(name, wheeldir):

pkgs/development/python-modules/bootstrapped-pip/prefix.patch

@@ -0,0 +1,115 @@
+diff --git a/pip/commands/install.py b/pip/commands/install.py
+index ddaa470..b798433 100644
+--- a/pip/commands/install.py
++++ b/pip/commands/install.py
+@@ -147,6 +147,13 @@ class InstallCommand(Command):
+                  "directory.")
+ 
+         cmd_opts.add_option(
++            '--prefix',
++            dest='prefix_path',
++            metavar='dir',
++            default=None,
++            help="Installation prefix where lib, bin and other top-level folders are placed")
++
++        cmd_opts.add_option(
+             "--compile",
+             action="store_true",
+             dest="compile",
+@@ -350,6 +357,7 @@ class InstallCommand(Command):
+                             install_options,
+                             global_options,
+                             root=options.root_path,
++                            prefix=options.prefix_path,
+                         )
+                         reqs = sorted(
+                             requirement_set.successfully_installed,
+diff --git a/pip/locations.py b/pip/locations.py
+index dfbc6da..b2f3383 100644
+--- a/pip/locations.py
++++ b/pip/locations.py
+@@ -209,7 +209,7 @@ site_config_files = [
+ 
+ 
+ def distutils_scheme(dist_name, user=False, home=None, root=None,
+-                     isolated=False):
++                     isolated=False, prefix=None):
+     """
+     Return a distutils install scheme
+     """
+@@ -231,6 +231,10 @@ def distutils_scheme(dist_name, user=False, home=None, root=None,
+     # or user base for installations during finalize_options()
+     # ideally, we'd prefer a scheme class that has no side-effects.
+     i.user = user or i.user
++    if user:
++        i.prefix = ""
++    else:
++        i.prefix = prefix or i.prefix
+     i.home = home or i.home
+     i.root = root or i.root
+     i.finalize_options()
+diff --git a/pip/req/req_install.py b/pip/req/req_install.py
+index 38013c5..14b868b 100644
+--- a/pip/req/req_install.py
++++ b/pip/req/req_install.py
+@@ -806,7 +806,7 @@ exec(compile(
+         else:
+             return True
+ 
+-    def install(self, install_options, global_options=(), root=None):
++    def install(self, install_options, global_options=[], root=None, prefix=None):
+         if self.editable:
+             self.install_editable(install_options, global_options)
+             return
+@@ -814,7 +814,7 @@ exec(compile(
+             version = pip.wheel.wheel_version(self.source_dir)
+             pip.wheel.check_compatibility(version, self.name)
+ 
+-            self.move_wheel_files(self.source_dir, root=root)
++            self.move_wheel_files(self.source_dir, root=root, prefix=prefix)
+             self.install_succeeded = True
+             return
+ 
+@@ -839,6 +839,8 @@ exec(compile(
+ 
+             if root is not None:
+                 install_args += ['--root', root]
++            if prefix is not None:
++                install_args += ['--prefix', prefix]
+ 
+             if self.pycompile:
+                 install_args += ["--compile"]
+@@ -1008,12 +1010,13 @@ exec(compile(
+     def is_wheel(self):
+         return self.link and self.link.is_wheel
+ 
+-    def move_wheel_files(self, wheeldir, root=None):
++    def move_wheel_files(self, wheeldir, root=None, prefix=None):
+         move_wheel_files(
+             self.name, self.req, wheeldir,
+             user=self.use_user_site,
+             home=self.target_dir,
+             root=root,
++            prefix=prefix,
+             pycompile=self.pycompile,
+             isolated=self.isolated,
+         )
+diff --git a/pip/wheel.py b/pip/wheel.py
+index 57246ca..738a6b0 100644
+--- a/pip/wheel.py
++++ b/pip/wheel.py
+@@ -130,12 +130,12 @@ def get_entrypoints(filename):
+ 
+ 
+ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None,
+-                     pycompile=True, scheme=None, isolated=False):
++                     pycompile=True, scheme=None, isolated=False, prefix=None):
+     """Install a wheel"""
+ 
+     if not scheme:
+         scheme = distutils_scheme(
+-            name, user=user, home=home, root=root, isolated=isolated
++            name, user=user, home=home, root=root, isolated=isolated, prefix=prefix,
+         )
+ 
+     if root_is_purelib(name, wheeldir):

pkgs/development/python-modules/generic/default.nix

@@ -3,7 +3,7 @@
    (http://pypi.python.org/pypi/setuptools/), which represents a large
    number of Python packages nowadays.  */
 
-{ python, setuptools, unzip, wrapPython, lib, recursivePthLoader, distutils-cfg }:
+{ python, setuptools, unzip, wrapPython, lib, bootstrapped-pip }:
 
 { name
 
@@ -12,28 +12,18 @@
 
 , buildInputs ? []
 
-# pass extra information to the distutils global configuration (think as global setup.cfg)
-, distutilsExtraCfg ? ""
-
 # propagate build dependencies so in case we have A -> B -> C,
 # C can import propagated packages by A
 , propagatedBuildInputs ? []
 
-# passed to "python setup.py install"
-, setupPyInstallFlags ? []
-
 # passed to "python setup.py build"
+# https://github.com/pypa/pip/issues/881
 , setupPyBuildFlags ? []
 
 # enable tests by default
 , doCheck ? true
 
-# List of packages that should be added to the PYTHONPATH
-# environment variable in programs built by this function.  Packages
-# in the standard `propagatedBuildInputs' variable are also added.
-# The difference is that `pythonPath' is not propagated to the user
-# environment.  This is preferrable for programs because it doesn't
-# pollute the user environment.
+# DEPRECATED: use propagatedBuildInputs
 , pythonPath ? []
 
 # used to disable derivation, useful for specific python versions
@@ -59,19 +49,19 @@ if disabled
 then throw "${name} not supported for interpreter ${python.executable}"
 else
 
+let
+  setuppy = "import setuptools, tokenize;__file__='setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))";
+in
 python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
   inherit doCheck;
 
   name = namePrefix + name;
 
-  buildInputs = [
-    wrapPython setuptools
-    (distutils-cfg.override { extraCfg = distutilsExtraCfg; })
-  ] ++ buildInputs ++ pythonPath
+  buildInputs = [ wrapPython bootstrapped-pip ] ++ buildInputs ++ pythonPath
     ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip);
 
   # propagate python/setuptools to active setup-hook in nix-shell
-  propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader python setuptools ];
+  propagatedBuildInputs = propagatedBuildInputs ++ [ python setuptools ];
 
   pythonPath = pythonPath;
 
@@ -79,85 +69,39 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
     runHook preConfigure
 
     # patch python interpreter to write null timestamps when compiling python files
-    # with following var we tell python to activate the patch so that python doesn't
-    # try to update them when we freeze timestamps in nix store
+    # this way python doesn't try to update them when we freeze timestamps in nix store
     export DETERMINISTIC_BUILD=1
 
-    # prepend following line to import setuptools before distutils
-    # this way we make sure setuptools monkeypatches distutils commands
-    # this way setuptools provides extra helpers such as "python setup.py test"
-    sed -i '0,/import distutils/s//import setuptools;import distutils/' setup.py
-    sed -i '0,/from distutils/s//import setuptools;from distutils/' setup.py
-
     runHook postConfigure
   '';
 
   checkPhase = attrs.checkPhase or ''
     runHook preCheck
-
-      ${python}/bin/${python.executable} setup.py test
-
+    ${python.interpreter} -c "${setuppy}" test
     runHook postCheck
   '';
 
   buildPhase = attrs.buildPhase or ''
     runHook preBuild
-
-    ${python}/bin/${python.executable} setup.py build ${lib.concatStringsSep " " setupPyBuildFlags}
-
+    ${python.interpreter} -c "${setuppy}" bdist_wheel
     runHook postBuild
   '';
 
   installPhase = attrs.installPhase or ''
     runHook preInstall
 
-    mkdir -p "$out/lib/${python.libPrefix}/site-packages"
+    mkdir -p "$out/${python.sitePackages}"
+    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
 
-    export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
-
-    ${python}/bin/${python.executable} setup.py install \
-      --install-lib=$out/lib/${python.libPrefix}/site-packages \
-      --old-and-unmanageable \
-      --prefix="$out" ${lib.concatStringsSep " " setupPyInstallFlags}
-
-    # --install-lib:
-    # sometimes packages specify where files should be installed outside the usual
-    # python lib prefix, we override that back so all infrastructure (setup hooks)
-    # work as expected
-
-    # --old-and-unmanagable:
-    # instruct setuptools not to use eggs but fallback to plan package install
-    # this also reduces one .pth file in the chain, but the main reason is to
-    # force install process to install only scripts for the package we are
-    # installing (otherwise it will install scripts also for dependencies)
-
-    # A pth file might have been generated to load the package from
-    # within its own site-packages, rename this package not to
-    # collide with others.
-    eapth="$out/lib/${python.libPrefix}"/site-packages/easy-install.pth
-    if [ -e "$eapth" ]; then
-        # move colliding easy_install.pth to specifically named one
-        mv "$eapth" $(dirname "$eapth")/${name}.pth
-    fi
-
-    # Remove any site.py files generated by easy_install as these
-    # cause collisions. If pth files are to be processed a
-    # corresponding site.py needs to be included in the PYTHONPATH.
-    rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
+    pushd dist
+    ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out
+    popd
 
     runHook postInstall
   '';
 
   postFixup = attrs.postFixup or ''
     wrapPythonPrograms
-
-      # TODO: document
-      createBuildInputsPth build-inputs "$buildInputStrings"
-      for inputsfile in propagated-build-inputs propagated-native-build-inputs; do
-        if test -e $out/nix-support/$inputsfile; then
-            createBuildInputsPth $inputsfile "$(cat $out/nix-support/$inputsfile)"
-        fi
-      done
   '';
 
   shellHook = attrs.shellHook or ''
@@ -178,5 +122,4 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
     # add extra maintainer(s) to every package
     maintainers = (meta.maintainers or []) ++ [ chaoflow iElectric ];
   };
-
 })

pkgs/top-level/python-packages.nix

@@ -126,6 +126,8 @@ let
     mpi = pkgs.openmpi;
   };
 
+  bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
+
   nixpart = callPackage ../tools/filesystems/nixpart { };
 
   # This is used for NixOps to make sure we won't break it with the next major
@@ -3349,11 +3351,11 @@ let
 
   decorator = buildPythonPackage rec {
     name = "decorator-${version}";
-    version = "3.4.2";
+    version = "4.0.4";
 
     src = pkgs.fetchurl {
       url = "http://pypi.python.org/packages/source/d/decorator/${name}.tar.gz";
-      sha256 = "7320002ce61dea6aa24adc945d9d7831b3669553158905cdd12f5d0027b54b44";
+      sha256 = "1qf3iiv401vhsdmf4bd08fwb3fq4xq769q2yl7zqqr1iml7w3l2s";
     };
 
     meta = {
@@ -10860,7 +10862,6 @@ let
       sha256 = "a39ce0e321e40e9758bf7b9128d316c71b35b80eabc84f13df492083bb6f1cc6";
     };
 
-    buildPhase = "${python}/bin/${python.executable} setup.py build";
     doCheck = false;
     postInstall = "ln -s $out/bin/osc-wrapper.py $out/bin/osc";
 
@@ -11110,6 +11111,10 @@ let
        sha256 = "0wf0k9xf5xzmi79418xq8zxwr7w7a4g4alv3dds9afb2l8bh9crg";
      };
 
+     patchPhase = ''
+       sed -i "s/test_gather_stats/noop/" futurist/tests/test_executors.py
+     '';
+
      propagatedBuildInputs = with self; [
        contextlib2 pbr six monotonic futures eventlet
      ];
@@ -12839,7 +12844,6 @@ let
     buildInputs = with self; [ python pkgs.libjpeg pkgs.zlib pkgs.freetype ];
 
     disabled = isPy3k;
-    doCheck = true;
 
     postInstall = "ln -s $out/lib/${python.libPrefix}/site-packages $out/lib/${python.libPrefix}/site-packages/PIL";
 
@@ -12857,7 +12861,6 @@ let
     '';
 
     checkPhase = "${python}/bin/${python.executable} selftest.py";
-    buildPhase = "${python}/bin/${python.executable} setup.py build_ext -i";
 
     meta = {
       homepage = http://www.pythonware.com/products/pil/;
@@ -15790,8 +15793,6 @@ let
 
     patches = [ ../development/python-modules/rpkg-buildfix.diff ];
 
-    # buildPhase = "python setup.py build";
-    # doCheck = false;
     propagatedBuildInputs = with self; [ pycurl pkgs.koji GitPython pkgs.git
                               pkgs.rpm pkgs.pyopenssl ];
 
@@ -18126,9 +18127,11 @@ let
     # # 1.0.0 and up create a circle dependency with traceback2/pbr
     doCheck = false;
 
-    # fixes a transient error when collecting tests, see https://bugs.launchpad.net/python-neutronclient/+bug/1508547
     patchPhase = ''
+      # # fixes a transient error when collecting tests, see https://bugs.launchpad.net/python-neutronclient/+bug/1508547
       sed -i '510i\        return None, False' unittest2/loader.py
+      # https://github.com/pypa/packaging/pull/36
+      sed -i 's/version=VERSION/version=str(VERSION)/' setup.py
     '';
 
     propagatedBuildInputs = with self; [ six argparse traceback2 ];
@@ -18204,7 +18207,7 @@ let
     name = "update_checker-0.11";
 
     src = pkgs.fetchurl {
-      url = "https://pypi.python.org/packages/source/u/update_checker/update_checker-0.11.tar.gz";
+      url = "https://pypi.python.org/packages/source/u/update_checker/${name}.tar.gz";
       md5 = "1daa54bac316be6624d7ee77373144bb";
     };