diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix index 82759ce7174b..db85d4ddd3c0 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix @@ -25,7 +25,6 @@ pkgs.stdenv.mkDerivation { buildPhase = '' runHook preBuild - ${python3.pkgs.black}/bin/black --quiet --check poetry2nix patchShebangs poetry2nix runHook postBuild ''; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index b6d533875523..0d1c243d0975 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -39,7 +39,11 @@ let lockData = readTOML poetrylock; lockFiles = lib.getAttrFromPath [ "metadata" "files" ] lockData; - specialAttrs = [ "poetrylock" "overrides" ]; + specialAttrs = [ + "overrides" + "poetrylock" + "pwd" + ]; passedAttrs = builtins.removeAttrs attrs specialAttrs; evalPep508 = mkEvalPep508 python; @@ -87,6 +91,8 @@ let inherit pkgs lib python poetryLib; }; poetry = poetryPkg; + # The canonical name is setuptools-scm + setuptools-scm = super.setuptools_scm; } ) # Null out any filtered packages, we don't want python.pkgs from nixpkgs @@ -149,7 +155,12 @@ let pyProject = readTOML pyproject; - specialAttrs = [ "pyproject" "poetrylock" "overrides" ]; + specialAttrs = [ + "overrides" + "poetrylock" + "pwd" + "pyproject" + ]; passedAttrs = builtins.removeAttrs attrs specialAttrs; getDeps = depAttr: let @@ -174,9 +185,9 @@ let format = "pyproject"; - nativeBuildInputs = [ pkgs.yj ]; buildInputs = mkInput "buildInputs" buildSystemPkgs; propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]); + nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj ]; checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies"); passthru = { @@ -186,7 +197,7 @@ let postPatch = (passedAttrs.postPatch or "") + '' # Tell poetry not to resolve the path dependencies. Any version is # fine ! - yj -tj < pyproject.toml | python ${./pyproject-without-path.py} > pyproject.json + yj -tj < pyproject.toml | ${python.interpreter} ${./pyproject-without-path.py} > pyproject.json yj -jt < pyproject.json > pyproject.toml rm pyproject.json ''; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index 95543ca73590..cbfa3dafa753 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -20,7 +20,8 @@ pythonPackages.callPackage ( { preferWheel ? false - }: + , ... + }@args: let @@ -82,7 +83,9 @@ pythonPackages.callPackage ( else (builtins.elemAt (lib.strings.splitString "-" name) 2); }; - baseBuildInputs = lib.optional (name != "setuptools_scm" && name != "setuptools-scm") pythonPackages.setuptools_scm; + baseBuildInputs = lib.optional (name != "setuptools_scm" && name != "setuptools-scm") pythonPackages.setuptools-scm; + + format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; in @@ -90,26 +93,34 @@ pythonPackages.callPackage ( pname = name; version = version; + inherit format; + doCheck = false; # We never get development deps - dontStrip = true; - format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; + + # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned` + dontStrip = format == "wheel"; nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []; - buildInputs = baseBuildInputs ++ (if !isSource then (getManyLinuxDeps fileInfo.name).pkg else []); + buildInputs = ( + baseBuildInputs + ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg + ++ lib.optional isLocal buildSystemPkgs + ); propagatedBuildInputs = - let - # Some dependencies like django gets the attribute name django - # but dependencies try to access Django - deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies); - in - (builtins.map (n: pythonPackages.${n}) deps) ++ (if isLocal then buildSystemPkgs else []); + # Some dependencies like django get the attribute name django + # but dependencies try to access Django + builtins.map (n: pythonPackages.${lib.toLower n}) (builtins.attrNames dependencies); meta = { - broken = ! isCompatible python.version python-versions; + broken = ! isCompatible python.pythonVersion python-versions; license = []; }; + passthru = { + inherit args; + }; + # We need to retrieve kind from the interpreter and the filename of the package # Interpreters should declare what wheel types they're compatible with (python type + ABI) # Here we can then choose a file based on that info. diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 48b8ff9859be..86260c037036 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -21,11 +21,15 @@ self: super: } ); - cffi = super.cffi.overrideAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.libffi ]; - } - ); + cffi = + # cffi is bundled with pypy + if self.python.implementation == "pypy" then null else ( + super.cffi.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libffi ]; + } + ) + ); cftime = super.cftime.overrideAttrs ( old: { @@ -60,9 +64,30 @@ self: super: } ); + dlib = super.dlib.overrideAttrs ( + old: { + # Parallel building enabled + inherit (pkgs.python.pkgs.dlib) patches; + + enableParallelBuilding = true; + dontUseCmakeConfigure = true; + + nativeBuildInputs = old.nativeBuildInputs ++ pkgs.dlib.nativeBuildInputs; + buildInputs = old.buildInputs ++ pkgs.dlib.buildInputs; + } + ); + # Environment markers are not always included (depending on how a dep was defined) enum34 = if self.pythonAtLeast "3.4" then null else super.enum34; + faker = super.faker.overrideAttrs ( + old: { + postPatch = '' + substituteInPlace setup.py --replace 'setup_requires=["pytest-runner"],' 'setup_requires=[],' || true + ''; + } + ); + grandalf = super.grandalf.overrideAttrs ( old: { postPatch = '' @@ -135,22 +160,39 @@ self: super: ); matplotlib = super.matplotlib.overrideAttrs ( - old: { - NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1"; + old: let + enableGhostscript = old.passthru.enableGhostscript or false; + enableGtk3 = old.passthru.enableTk or false; + enableQt = old.passthru.enableQt or false; + enableTk = old.passthru.enableTk or false; - XDG_RUNTIME_DIR = "/tmp"; + inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa; - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.pkgconfig - ]; + in + { + NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1"; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - pkgs.libpng - pkgs.freetype - ]; + XDG_RUNTIME_DIR = "/tmp"; - inherit (super.matplotlib) patches; - } + buildInputs = old.buildInputs + ++ lib.optional enableGhostscript pkgs.ghostscript + ++ lib.optional stdenv.isDarwin [ Cocoa ]; + + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + ]; + + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.libpng + pkgs.freetype + ] + ++ stdenv.lib.optionals enableGtk3 [ pkgs.cairo self.pycairo pkgs.gtk3 pkgs.gobject-introspection self.pygobject3 ] + ++ stdenv.lib.optionals enableTk [ pkgs.tcl pkgs.tk self.tkinter pkgs.libX11 ] + ++ stdenv.lib.optionals enableQt [ self.pyqt5 ] + ; + + inherit (super.matplotlib) patches; + } ); # Calls Cargo at build time for source builds and is really tricky to package @@ -191,7 +233,7 @@ self: super: numpy = super.numpy.overrideAttrs ( old: let - blas = pkgs.openblasCompat; + blas = old.passthru.args.blas or pkgs.openblasCompat; blasImplementation = lib.nameFromURL blas.name "-"; cfg = pkgs.writeTextFile { name = "site.cfg"; @@ -222,6 +264,19 @@ self: super: } ); + peewee = super.peewee.overridePythonAttrs ( + old: let + withPostgres = old.passthru.withPostgres or false; + withMysql = old.passthru.withMysql or false; + in + { + buildInputs = old.buildInputs ++ [ self.cython pkgs.sqlite ]; + propagatedBuildInputs = old.propagatedBuildInputs + ++ lib.optional withPostgres self.psycopg2 + ++ lib.optional withMysql self.mysql-connector; + } + ); + pillow = super.pillow.overrideAttrs ( old: { nativeBuildInputs = [ pkgs.pkgconfig ] ++ old.nativeBuildInputs; @@ -297,84 +352,106 @@ self: super: } ); - pyqt5 = super.pyqt5.overridePythonAttrs ( - old: { - format = "other"; + pyqt5 = let + drv = super.pyqt5; + withConnectivity = drv.passthru.args.withConnectivity or false; + withMultimedia = drv.passthru.args.withMultimedia or false; + withWebKit = drv.passthru.args.withWebKit or false; + withWebSockets = drv.passthru.args.withWebSockets or false; + in + super.pyqt5.overridePythonAttrs ( + old: { + format = "other"; - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.pkgconfig - pkgs.qt5.qmake - pkgs.xorg.lndir - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - pkgs.qt5.qtwebchannel - # self.pyqt5-sip - self.sip - ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + pkgs.qt5.qmake + pkgs.xorg.lndir + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + pkgs.qt5.qtwebchannel + # self.pyqt5-sip + self.sip + ] + ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity + ++ lib.optional withMultimedia pkgs.qt5.qtmultimedia + ++ lib.optional withWebKit pkgs.qt5.qtwebkit + ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets + ; - buildInputs = old.buildInputs ++ [ - pkgs.dbus - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - self.sip - ]; + buildInputs = old.buildInputs ++ [ + pkgs.dbus + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + self.sip + ] + ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity + ++ lib.optional withWebKit pkgs.qt5.qtwebkit + ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets + ; - # Fix dbus mainloop - inherit (pkgs.python3.pkgs.pyqt5) patches; + # Fix dbus mainloop + patches = pkgs.python3.pkgs.pyqt5.patches or []; - configurePhase = '' - runHook preConfigure + configurePhase = '' + runHook preConfigure - export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} + export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} - mkdir -p $out/${self.python.sitePackages}/dbus/mainloop - ${self.python.executable} configure.py -w \ - --confirm-license \ - --no-qml-plugin \ - --bindir=$out/bin \ - --destdir=$out/${self.python.sitePackages} \ - --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ - --sipdir=$out/share/sip/PyQt5 \ - --designer-plugindir=$out/plugins/designer + mkdir -p $out/${self.python.sitePackages}/dbus/mainloop + ${self.python.executable} configure.py -w \ + --confirm-license \ + --no-qml-plugin \ + --bindir=$out/bin \ + --destdir=$out/${self.python.sitePackages} \ + --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ + --sipdir=$out/share/sip/PyQt5 \ + --designer-plugindir=$out/plugins/designer - runHook postConfigure - ''; - - postInstall = '' - ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ - for i in $out/bin/*; do - wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" - done - - # # Let's make it a namespace package - # cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py - # from pkgutil import extend_path - # __path__ = extend_path(__path__, __name__) - # EOF - ''; - - installCheckPhase = let - modules = [ - "PyQt5" - "PyQt5.QtCore" - "PyQt5.QtQml" - "PyQt5.QtWidgets" - "PyQt5.QtGui" - ]; - imports = lib.concatMapStrings (module: "import ${module};") modules; - in - '' - echo "Checking whether modules can be imported..." - ${self.python.interpreter} -c "${imports}" + runHook postConfigure ''; - doCheck = true; + postInstall = '' + ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ + for i in $out/bin/*; do + wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" + done - enableParallelBuilding = true; - } - ); + # Let's make it a namespace package + cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + EOF + ''; + + installCheckPhase = let + modules = [ + "PyQt5" + "PyQt5.QtCore" + "PyQt5.QtQml" + "PyQt5.QtWidgets" + "PyQt5.QtGui" + ] + ++ lib.optional withWebSockets "PyQt5.QtWebSockets" + ++ lib.optional withWebKit "PyQt5.QtWebKit" + ++ lib.optional withMultimedia "PyQt5.QtMultimedia" + ++ lib.optional withConnectivity "PyQt5.QtConnectivity" + ; + + imports = lib.concatMapStrings (module: "import ${module};") modules; + in + '' + echo "Checking whether modules can be imported..." + ${self.python.interpreter} -c "${imports}" + ''; + + doCheck = true; + + enableParallelBuilding = true; + } + ); pytest-datadir = super.pytest-datadir.overrideAttrs ( old: { @@ -400,9 +477,21 @@ self: super: } ); + # Pybind11 is an undeclared dependency of scipy that we need to pick from nixpkgs + # Make it not fail with infinite recursion + pybind11 = super.pybind11.overridePythonAttrs ( + old: { + cmakeFlags = (old.cmakeFlags or []) ++ [ + "-DPYBIND11_TEST=off" + ]; + doCheck = false; # Circular test dependency + } + ); + scipy = super.scipy.overrideAttrs ( old: { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pybind11 ]; setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; enableParallelBuilding = true; buildInputs = old.buildInputs ++ [ self.numpy.blas ]; @@ -416,10 +505,27 @@ self: super: } ); + scikit-learn = super.scikit-learn.overridePythonAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + pkgs.gfortran + pkgs.glibcLocales + ] ++ lib.optionals stdenv.cc.isClang [ + pkgs.llvmPackages.openmp + ]; + + nativeBuildInputs = old.nativeBuildInputs ++ [ + self.cython + ]; + + enableParallelBuilding = true; + } + ); + shapely = super.shapely.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ pkgs.geos self.cython ]; - inherit (super.shapely) patches GEOS_LIBRARY_PATH; + inherit (pkgs.python3.pkgs.shapely) patches GEOS_LIBRARY_PATH; } ); @@ -431,7 +537,7 @@ self: super: } ); - vose-alias-method = super.pytest-datadir.overrideAttrs ( + vose-alias-method = super.vose-alias-method.overrideAttrs ( old: { postInstall = '' rm -f $out/LICENSE @@ -439,6 +545,15 @@ self: super: } ); + uvloop = super.uvloop.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ lib.optionals stdenv.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.ApplicationServices + pkgs.darwin.apple_sdk.frameworks.CoreServices + ]; + } + ); + # Stop infinite recursion by using bootstrapped pkg from nixpkgs wheel = ( pkgs.python3.pkgs.override { @@ -450,4 +565,17 @@ self: super: } ); + zipp = + if lib.versionAtLeast super.zipp.version "2.0.0" then ( + super.zipp.overridePythonAttrs ( + old: { + prePatch = '' + substituteInPlace setup.py --replace \ + 'setuptools.setup()' \ + 'setuptools.setup(version="${super.zipp.version}")' + ''; + } + ) + ) else super.zipp; + } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix index 93a395326eb7..ae0c29f36838 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix @@ -83,14 +83,24 @@ let # This function also performs variable substitution, replacing environment markers with their explicit values transformExpressions = exprs: let variables = { - os_name = "posix"; # TODO: Check other platforms + os_name = ( + if python.pname == "jython" then "java" + else "posix" + ); sys_platform = ( if stdenv.isLinux then "linux" else if stdenv.isDarwin then "darwin" else throw "Unsupported platform" ); platform_machine = stdenv.platform.kernelArch; - platform_python_implementation = "CPython"; # Only CPython supported for now + platform_python_implementation = let + impl = python.passthru.implementation; + in + ( + if impl == "cpython" then "CPython" + else if impl == "pypy" then "PyPy" + else throw "Unsupported implementation ${impl}" + ); platform_release = ""; # Field not reproducible platform_system = ( if stdenv.isLinux then "Linux" @@ -100,7 +110,7 @@ let platform_version = ""; # Field not reproducible python_version = python.passthru.pythonVersion; python_full_version = python.version; - implementation_name = "cpython"; # Only cpython supported for now + implementation_name = python.implementation; implementation_version = python.version; extra = ""; };