Refactor buildPythonPackage to modularize building process.
Before we used `easy_install` command to handle installation in one shot, now this is split into two phases: - buildPhase: python setup.py build - installPhase: python setup.py install Each of those commands have the ability to pass extra parameters through buildPythonPackage parameters as `setupPyInstallFlags` and `setupPyBuildFlags`. Phases now correctly execute post/pre hooks. In configurePhase we inject setuptools dependency before distutils is imported to apply monkeypatching by setuptools that is needed for special features to apply. We don't have to reorder default phases anymore, as test phase comes after build and that works. I rewrote offineDistutils into distutils-cfg with a bit cleaner syntax and ability to specify extraCfg to the config file. Plone packages are failing and garbas said he will adopt them to the new functions. The rest of the packages I fixed and these commits shouldn't break any package (according to my testings) and they introduce 16 new jobs and fix 38 that were broken before.
This commit is contained in:
Domen Kožar
parent
e9923c6499
commit
bf5d6fb9b1
31
pkgs/development/python-modules/distutils-cfg/default.nix
Normal file
31
pkgs/development/python-modules/distutils-cfg/default.nix
Normal file
@ -0,0 +1,31 @@
|
||||
# global distutils configuration, see http://docs.python.org/2/install/index.html#distutils-configuration-files
|
||||
|
||||
{ stdenv, python, writeText, extraCfg ? "" }:
|
||||
|
||||
|
||||
let
|
||||
distutilsCfg = writeText "distutils.cfg" ''
|
||||
[easy_install]
|
||||
|
||||
# don't allow network connections during build to ensure purity
|
||||
allow-hosts = None
|
||||
|
||||
# make sure we always unzip installed packages otherwise setup hooks won't work
|
||||
zip_ok = 0
|
||||
|
||||
${extraCfg}
|
||||
'';
|
||||
in stdenv.mkDerivation {
|
||||
name = "distutils.cfg-python${python.version}";
|
||||
|
||||
buildInputs = [ python ];
|
||||
|
||||
unpackPhase = "true";
|
||||
|
||||
installPhase = ''
|
||||
dest="$out/lib/${python.libPrefix}/site-packages/distutils"
|
||||
mkdir -p $dest
|
||||
ln -s ${python}/lib/${python.libPrefix}/distutils/* $dest
|
||||
ln -s ${distutilsCfg} $dest/distutils.cfg
|
||||
'';
|
||||
}
|
||||
@ -1,84 +1,95 @@
|
||||
/* This function provides a generic Python package builder. It is
|
||||
intended to work with packages that use `setuptools'
|
||||
intended to work with packages that use `distutils/setuptools'
|
||||
(http://pypi.python.org/pypi/setuptools/), which represents a large
|
||||
number of Python packages nowadays. */
|
||||
|
||||
{ python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }:
|
||||
{ python, setuptools, wrapPython, lib, recursivePthLoader, distutils-cfg }:
|
||||
|
||||
{ name, namePrefix ? python.libPrefix + "-"
|
||||
{ name
|
||||
|
||||
# by default prefix name with python version name, e.g. "python3.3-"
|
||||
, namePrefix ? python.libPrefix + "-"
|
||||
|
||||
, buildInputs ? []
|
||||
|
||||
# TODO: document
|
||||
, distutilsExtraCfg ? ""
|
||||
|
||||
# TODO: say what it does
|
||||
, propagatedBuildInputs ? []
|
||||
|
||||
, # List of packages that should be added to the PYTHONPATH
|
||||
# environment variable in programs built by this function. Packages
|
||||
# in the standard `propagatedBuildInputs' variable are also added.
|
||||
# The difference is that `pythonPath' is not propagated to the user
|
||||
# environment. This is preferrable for programs because it doesn't
|
||||
# pollute the user environment.
|
||||
pythonPath ? []
|
||||
# passed to "python setup.py install"
|
||||
, setupPyInstallFlags ? []
|
||||
|
||||
, installCommand ?
|
||||
''
|
||||
easy_install --always-unzip --prefix="$out" .
|
||||
''
|
||||
|
||||
, preConfigure ? "true"
|
||||
|
||||
, buildPhase ? "true"
|
||||
# passed to "python setup.py build"
|
||||
, setupPyBuildFlags ? []
|
||||
|
||||
# enable tests by default
|
||||
, doCheck ? true
|
||||
|
||||
, checkPhase ?
|
||||
''
|
||||
runHook preCheck
|
||||
${python}/bin/${python.executable} setup.py test
|
||||
runHook postCheck
|
||||
''
|
||||
# List of packages that should be added to the PYTHONPATH
|
||||
# environment variable in programs built by this function. Packages
|
||||
# in the standard `propagatedBuildInputs' variable are also added.
|
||||
# The difference is that `pythonPath' is not propagated to the user
|
||||
# environment. This is preferrable for programs because it doesn't
|
||||
# pollute the user environment.
|
||||
, pythonPath ? []
|
||||
|
||||
, preInstall ? ""
|
||||
, postInstall ? ""
|
||||
|
||||
, meta ? {}
|
||||
, meta ? {}
|
||||
|
||||
, ... } @ attrs:
|
||||
|
||||
# Keep extra attributes from ATTR, e.g., `patchPhase', etc.
|
||||
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
|
||||
python.stdenv.mkDerivation (attrs // {
|
||||
inherit doCheck buildPhase checkPhase;
|
||||
inherit doCheck;
|
||||
|
||||
name = namePrefix + name;
|
||||
|
||||
# default to python's platforms and add maintainer(s) to every
|
||||
# package
|
||||
meta = {
|
||||
platforms = python.meta.platforms;
|
||||
} // meta // {
|
||||
maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
|
||||
};
|
||||
|
||||
# checkPhase after installPhase to run tests on installed packages
|
||||
phases = "unpackPhase patchPhase configurePhase buildPhase installPhase checkPhase fixupPhase distPhase";
|
||||
|
||||
buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath;
|
||||
buildInputs = [ python wrapPython setuptools (distutils-cfg.override { extraCfg = distutilsExtraCfg; }) ] ++ buildInputs ++ pythonPath;
|
||||
|
||||
propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader ];
|
||||
|
||||
pythonPath = [ setuptools ] ++ pythonPath;
|
||||
|
||||
preConfigure = ''
|
||||
configurePhase = attrs.configurePhase or ''
|
||||
runHook preConfigure
|
||||
|
||||
# TODO: document
|
||||
export DETERMINISTIC_BUILD=1
|
||||
PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
|
||||
${preConfigure}
|
||||
|
||||
# we need to prepend following line to monkeypatch distutils commands
|
||||
sed -i '0,/import distutils/s//import setuptools;import distutils/' setup.py
|
||||
sed -i '0,/from distutils/s//import setuptools;from distutils/' setup.py
|
||||
|
||||
|
||||
runHook postConfigure
|
||||
'';
|
||||
|
||||
installPhase = preInstall + ''
|
||||
checkPhase = attrs.checkPhase or ''
|
||||
runHook preCheck
|
||||
|
||||
${python}/bin/${python.executable} setup.py test -q
|
||||
|
||||
runHook postCheck
|
||||
'';
|
||||
|
||||
buildPhase = attrs.buildPhase or ''
|
||||
runHook preBuild
|
||||
|
||||
${python}/bin/${python.executable} setup.py build ${lib.concatStringsSep " " setupPyBuildFlags}
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
installPhase = attrs.installPhase or ''
|
||||
runHook preInstall
|
||||
|
||||
mkdir -p "$out/lib/${python.libPrefix}/site-packages"
|
||||
|
||||
echo "installing \`${name}' with \`easy_install'..."
|
||||
export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
|
||||
${installCommand}
|
||||
|
||||
${python}/bin/${python.executable} setup.py install --install-lib=$out/lib/${python.libPrefix}/site-packages \
|
||||
--prefix="$out" ${lib.concatStringsSep " " setupPyInstallFlags}
|
||||
|
||||
# A pth file might have been generated to load the package from
|
||||
# within its own site-packages, rename this package not to
|
||||
@ -94,17 +105,18 @@ python.stdenv.mkDerivation (attrs // {
|
||||
# corresponding site.py needs to be included in the PYTHONPATH.
|
||||
rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
|
||||
|
||||
${postInstall}
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
postFixup =
|
||||
''
|
||||
wrapPythonPrograms
|
||||
|
||||
# If a user installs a Python package, she probably also wants its
|
||||
# If a user installs a Python package, they probably also wants its
|
||||
# dependencies in the user environment (since Python modules don't
|
||||
# have something like an RPATH, so the only way to find the
|
||||
# dependencies is to have them in the PYTHONPATH variable).
|
||||
# TODO: better docs
|
||||
if test -e $out/nix-support/propagated-build-inputs; then
|
||||
ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages
|
||||
fi
|
||||
@ -116,4 +128,13 @@ python.stdenv.mkDerivation (attrs // {
|
||||
fi
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# default to python's platforms
|
||||
platforms = python.meta.platforms;
|
||||
} // meta // {
|
||||
# add extra maintainer(s) to every package
|
||||
maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
|
||||
};
|
||||
|
||||
})
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
# Used during module installation to prevent easy_install and python
|
||||
# setup.py install/test from downloading
|
||||
|
||||
{ stdenv, python }:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "python-offline-distutils-${python.version}";
|
||||
|
||||
buildInputs = [ python ];
|
||||
|
||||
unpackPhase = "true";
|
||||
installPhase = ''
|
||||
dst="$out/lib/${python.libPrefix}/site-packages"
|
||||
ensureDir $dst/distutils
|
||||
ln -s ${python}/lib/${python.libPrefix}/distutils/* $dst/distutils/
|
||||
cat <<EOF > $dst/distutils/distutils.cfg
|
||||
[easy_install]
|
||||
allow-hosts = None
|
||||
EOF
|
||||
'';
|
||||
}
|
||||
@ -20,10 +20,10 @@ pythonPackages = modules // import ./python-packages-generated.nix {
|
||||
|
||||
callPackage = pkgs.lib.callPackageWith (pkgs // pythonPackages);
|
||||
|
||||
buildPythonPackage = import ../development/python-modules/generic {
|
||||
inherit (pkgs) lib;
|
||||
inherit python wrapPython setuptools recursivePthLoader offlineDistutils;
|
||||
};
|
||||
# global distutils config used by buildPythonPackage
|
||||
distutils-cfg = callPackage ../development/python-modules/distutils-cfg { };
|
||||
|
||||
buildPythonPackage = callPackage ../development/python-modules/generic { };
|
||||
|
||||
wrapPython = pkgs.makeSetupHook
|
||||
{ deps = pkgs.makeWrapper;
|
||||
@ -33,11 +33,6 @@ pythonPackages = modules // import ./python-packages-generated.nix {
|
||||
|
||||
# specials
|
||||
|
||||
offlineDistutils = import ../development/python-modules/offline-distutils {
|
||||
inherit (pkgs) stdenv;
|
||||
inherit python;
|
||||
};
|
||||
|
||||
recursivePthLoader = import ../development/python-modules/recursive-pth-loader {
|
||||
inherit (pkgs) stdenv;
|
||||
inherit python;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user