From d3f756d9b7e0d8126a4f2a28d267854a0e7bb141 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 2 Mar 2020 16:23:25 +0000 Subject: [PATCH] poetry2nix: 1.5.0 -> 1.6.0 --- .../tools/poetry2nix/poetry2nix/default.nix | 113 ++++++++--------- .../poetry2nix/poetry2nix/hooks/default.nix | 8 ++ .../poetry2nix/poetry2nix/hooks/fixup-hook.sh | 8 ++ .../hooks/remove-path-dependencies.sh | 4 + .../tools/poetry2nix/poetry2nix/lib.nix | 33 +++++ .../poetry2nix/poetry2nix/mk-poetry-dep.nix | 26 ++-- .../tools/poetry2nix/poetry2nix/overrides.nix | 115 ++++++++++++++++-- 7 files changed, 237 insertions(+), 70 deletions(-) create mode 100644 pkgs/development/tools/poetry2nix/poetry2nix/hooks/fixup-hook.sh diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index ac866a139d97..7d3164fcec62 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -3,12 +3,11 @@ , poetry ? null , poetryLib ? import ./lib.nix { inherit lib pkgs; } }: - let inherit (poetryLib) isCompatible readTOML; # Poetry2nix version - version = "1.1.0"; + version = "1.6.0"; /* The default list of poetry2nix override overlays */ defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; }); @@ -29,14 +28,15 @@ let Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile. */ mkPoetryPackages = - { pyproject - , poetrylock - , poetryPkg + { projectDir ? null + , pyproject ? projectDir + "/pyproject.toml" + , poetrylock ? projectDir + "/poetry.lock" , overrides ? [ defaultPoetryOverrides ] - , meta ? {} , python ? pkgs.python3 - , pwd ? null + , pwd ? projectDir }@attrs: let + poetryPkg = poetry.override { inherit python; }; + pyProject = readTOML pyproject; poetryLock = readTOML poetrylock; lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock; @@ -52,14 +52,7 @@ let # Filter packages by their PEP508 markers & pyproject interpreter version partitions = let - supportsPythonVersion = pkgMeta: let - pep508Result = if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; - - flatDeps = (pyProject.tool.poetry.dependencies or {}) // (pyProject.tool.poetry.dev-dependencies or {}); - constraints = flatDeps.${pkgMeta.name}.python or ""; - pyprojectResult = isCompatible python.pythonVersion constraints; - in - pyprojectResult && pep508Result; + supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; in lib.partition supportsPythonVersion poetryLock.package; @@ -105,7 +98,7 @@ let # The canonical name is setuptools-scm setuptools-scm = super.setuptools_scm; - inherit (hooks) removePathDependenciesHook; + inherit (hooks) removePathDependenciesHook poetry2nixFixupHook; } ) # Null out any filtered packages, we don't want python.pkgs from nixpkgs @@ -133,18 +126,17 @@ let poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; } */ mkPoetryEnv = - { pyproject - , poetrylock + { projectDir ? null + , pyproject ? projectDir + "/pyproject.toml" + , poetrylock ? projectDir + "/poetry.lock" , overrides ? [ defaultPoetryOverrides ] - , meta ? {} - , pwd ? null + , pwd ? projectDir , python ? pkgs.python3 }: let - poetryPkg = poetry.override { inherit python; }; py = mkPoetryPackages ( { - inherit poetryPkg pyproject poetrylock overrides meta python pwd; + inherit pyproject poetrylock overrides python pwd; } ); in @@ -152,19 +144,18 @@ let /* Creates a Python application from pyproject.toml and poetry.lock */ mkPoetryApplication = - { src - , pyproject - , poetrylock + { projectDir ? null + , src ? poetryLib.cleanPythonSources { src = projectDir; } + , pyproject ? projectDir + "/pyproject.toml" + , poetrylock ? projectDir + "/poetry.lock" , overrides ? [ defaultPoetryOverrides ] , meta ? {} , python ? pkgs.python3 - , pwd ? null + , pwd ? projectDir , ... }@attrs: let - poetryPkg = poetry.override { inherit python; }; - poetryPython = mkPoetryPackages { - inherit poetryPkg pyproject poetrylock overrides meta python pwd; + inherit pyproject poetrylock overrides python pwd; }; py = poetryPython.python; @@ -178,11 +169,20 @@ let ]; passedAttrs = builtins.removeAttrs attrs specialAttrs; + # Get dependencies and filter out depending on interpreter version getDeps = depAttr: let + compat = isCompatible py.pythonVersion; deps = pyProject.tool.poetry.${depAttr} or {}; depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps); in - builtins.map (dep: py.pkgs."${dep}") depAttrs; + builtins.map ( + dep: let + pkg = py.pkgs."${dep}"; + constraints = deps.${dep}.python or ""; + isCompat = compat constraints; + in + if isCompat then pkg else null + ) depAttrs; getInputs = attr: attrs.${attr} or []; mkInput = attr: extraInputs: getInputs attr ++ extraInputs; @@ -198,6 +198,8 @@ let pname = pyProject.tool.poetry.name; version = pyProject.tool.poetry.version; + inherit src; + format = "pyproject"; buildInputs = mkInput "buildInputs" buildSystemPkgs; @@ -211,6 +213,7 @@ let meta = meta // { inherit (pyProject.tool.poetry) description homepage; + inherit (py.meta) platforms; license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown"); }; @@ -220,34 +223,11 @@ let /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies */ cli = import ./cli.nix { inherit pkgs lib version; }; - /* Poetry2nix documentation */ - doc = pkgs.stdenv.mkDerivation { - pname = "poetry2nix-docs"; - inherit version; - - src = pkgs.runCommandNoCC "poetry2nix-docs-src" {} '' - mkdir -p $out - cp ${./default.nix} $out/default.nix - ''; - - buildInputs = [ - pkgs.nixdoc - ]; - - buildPhase = '' - nixdoc --category poetry2nix --description "Poetry2nix functions" --file ./default.nix > poetry2nix.xml - ''; - - installPhase = '' - mkdir -p $out - cp poetry2nix.xml $out/ - ''; - - }; - in { - inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli doc; + inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli version; + + inherit (poetryLib) cleanPythonSources; /* The default list of poetry2nix override overlays @@ -262,4 +242,25 @@ in in defaultSet // customSet; }; + + /* + Convenience functions for specifying overlays with or without the poerty2nix default overrides + */ + overrides = { + /* + Returns the specified overlay in a list + */ + withoutDefaults = overlay: [ + overlay + ]; + + /* + Returns the specified overlay and returns a list + combining it with poetry2nix default overrides + */ + withDefaults = overlay: [ + defaultPoetryOverrides + overlay + ]; + }; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix index dd24b2634e00..ec3fa0afa69d 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix @@ -22,4 +22,12 @@ in } ./remove-path-dependencies.sh ) {}; + poetry2nixFixupHook = callPackage ( + {}: + makeSetupHook { + name = "fixup-hook.sh"; + deps = []; + } ./fixup-hook.sh + ) {}; + } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/fixup-hook.sh b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/fixup-hook.sh new file mode 100644 index 000000000000..fc539e4298c6 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/fixup-hook.sh @@ -0,0 +1,8 @@ +poetry2nix-fixup-hook() { + # Including tests in the output is a common mistake + if [ -z "${dontFixupTests-}" ]; then + rm -rf $out/lib/python3.7/site-packages/tests + fi +} + +postFixupHooks+=(poetry2nix-fixup-hook) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh index 900fe66908bf..b22a05900a05 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh @@ -1,4 +1,8 @@ remove-path-dependencies-hook() { + if ! test -f pyproject.toml; then + return + fi + # Tell poetry not to resolve the path dependencies. Any version is fine! @yj@ -tj < pyproject.toml | @pythonInterpreter@ @pyprojectPatchScript@ > pyproject.json @yj@ -jt < pyproject.json > pyproject.toml diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix index 64ba0293132e..b816feb38e89 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -96,6 +96,38 @@ let [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ] ); + # Find gitignore files recursively in parent directory stopping with .git + findGitIgnores = path: let + parent = path + "/.."; + gitIgnore = path + "/.gitignore"; + isGitRoot = builtins.pathExists (path + "/.git"); + hasGitIgnore = builtins.pathExists gitIgnore; + gitIgnores = if hasGitIgnore then [ gitIgnore ] else []; + in + lib.optionals (builtins.toString path != "/" && ! isGitRoot) (findGitIgnores parent) ++ gitIgnores; + + /* + Provides a source filtering mechanism that: + + - Filters gitignore's + - Filters pycache/pyc files + - Uses cleanSourceFilter to filter out .git/.hg, .o/.so, editor backup files & nix result symlinks + */ + cleanPythonSources = { src }: let + gitIgnores = findGitIgnores src; + pycacheFilter = name: type: + (type == "directory" && ! lib.strings.hasInfix "__pycache__" name) + || (type == "regular" && ! lib.strings.hasSuffix ".pyc" name) + ; + in + lib.cleanSourceWith { + filter = lib.cleanSourceFilter; + src = lib.cleanSourceWith { + filter = pkgs.nix-gitignore.gitignoreFilterPure pycacheFilter gitIgnores src; + inherit src; + }; + }; + in { inherit @@ -105,5 +137,6 @@ in readTOML getBuildSystemPkgs satisfiesSemver + cleanPythonSources ; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index 5113ad25e4b6..5ae35dfed49e 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -106,8 +106,10 @@ pythonPackages.callPackage ( # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned` dontStrip = format == "wheel"; - nativeBuildInputs = (if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []) - ++ lib.optional (isLocal) pkgs.yj + nativeBuildInputs = [ + pythonPackages.poetry2nixFixupHook + ] + ++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook ; @@ -117,14 +119,24 @@ pythonPackages.callPackage ( ++ lib.optional isLocal buildSystemPkgs ); - propagatedBuildInputs = - # Some dependencies like django get the attribute name django - # but dependencies try to access Django - builtins.map (n: pythonPackages.${lib.toLower n}) (builtins.attrNames dependencies); + propagatedBuildInputs = let + compat = isCompatible python.pythonVersion; + deps = lib.filterAttrs (n: v: v) ( + lib.mapAttrs ( + n: v: let + constraints = v.python or ""; + in + compat constraints + ) dependencies + ); + depAttrs = lib.attrNames deps; + in + builtins.map (n: pythonPackages.${lib.toLower n}) depAttrs; meta = { broken = ! isCompatible python.pythonVersion python-versions; license = []; + inherit (python.meta) platforms; }; passthru = { @@ -139,7 +151,7 @@ pythonPackages.callPackage ( inherit (source) url; rev = source.reference; } - ) else if isLocal then (localDepPath) else fetchFromPypi { + ) else if isLocal then (poetryLib.cleanPythonSources { src = localDepPath; }) else fetchFromPypi { pname = name; inherit (fileInfo) file hash kind; }; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 6316dcf51c0c..9eb3f92bd12b 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -6,6 +6,13 @@ self: super: { + astroid = super.astroid.overrideAttrs ( + old: rec { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); + av = super.av.overrideAttrs ( old: { nativeBuildInputs = old.nativeBuildInputs ++ [ @@ -81,17 +88,36 @@ self: super: enum34 = if self.pythonAtLeast "3.4" then null else super.enum34; faker = super.faker.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); + + fancycompleter = super.fancycompleter.overrideAttrs ( old: { postPatch = '' - substituteInPlace setup.py --replace 'setup_requires=["pytest-runner"],' 'setup_requires=[],' || true + substituteInPlace setup.py \ + --replace 'setup_requires="setupmeta"' 'setup_requires=[]' \ + --replace 'versioning="devcommit"' 'version="${old.version}"' ''; } ); grandalf = super.grandalf.overrideAttrs ( old: { - postPatch = '' - substituteInPlace setup.py --replace "setup_requires=['pytest-runner',]," "setup_requires=[]," || true + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); + + h5py = super.h5py.overrideAttrs ( + old: rec { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ]; + configure_flags = "--hdf5=${pkgs.hdf5}"; + postConfigure = '' + ${self.python.executable} setup.py configure ${configure_flags} ''; } ); @@ -103,12 +129,21 @@ self: super: ); # importlib-metadata has an incomplete dependency specification - importlib-metadata = if super.importlib-metadata == null then null else super.importlib-metadata.overrideAttrs ( + importlib-metadata = super.importlib-metadata.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2; } ); + jupyter = super.jupyter.overrideAttrs ( + old: rec { + # jupyter is a meta-package. Everything relevant comes from the + # dependencies. It does however have a jupyter.py file that conflicts + # with jupyter-core so this meta solves this conflict. + meta.priority = 100; + } + ); + lap = super.lap.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ [ @@ -202,9 +237,8 @@ self: super: mccabe = super.mccabe.overrideAttrs ( old: { - postPatch = '' - substituteInPlace setup.py --replace "setup_requires=['pytest-runner']," "setup_requires=[]," || true - ''; + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; } ); @@ -264,6 +298,13 @@ self: super: } ); + openexr = super.openexr.overrideAttrs ( + old: rec { + buildInputs = old.buildInputs ++ [ pkgs.openexr pkgs.ilmbase ]; + NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ]; + } + ); + peewee = super.peewee.overridePythonAttrs ( old: let withPostgres = old.passthru.withPostgres or false; @@ -346,6 +387,13 @@ self: super: } ); + pylint = super.pylint.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); + pyopenssl = super.pyopenssl.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ pkgs.openssl ]; @@ -461,6 +509,14 @@ self: super: } ); + pytest = super.pytest.overridePythonAttrs ( + old: { + doCheck = false; + } + ); + + pytest-runner = super.pytest-runner or super.pytestrunner; + python-prctl = super.python-prctl.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ @@ -469,6 +525,21 @@ self: super: } ); + pyzmq = super.pyzmq.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.zeromq ]; + } + ); + + rockset = super.rockset.overrideAttrs ( + old: rec { + postPatch = '' + cp ./setup_rockset.py ./setup.py + ''; + } + ); + scaleapi = super.scaleapi.overrideAttrs ( old: { postPatch = '' @@ -477,6 +548,12 @@ self: super: } ); + pandas = super.pandas.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ self.cython ]; + } + ); + # Pybind11 is an undeclared dependency of scipy that we need to pick from nixpkgs # Make it not fail with infinite recursion pybind11 = super.pybind11.overridePythonAttrs ( @@ -529,6 +606,30 @@ self: super: } ); + shellingham = if lib.versionAtLeast super.shellingham.version "1.3.2" then ( + super.shellingham.overridePythonAttrs ( + old: { + format = "pyproject"; + } + ) + ) else super.shellingham; + + tables = super.tables.overrideAttrs ( + old: { + HDF5_DIR = "${pkgs.hdf5}"; + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + propagatedBuildInputs = old.nativeBuildInputs ++ [ pkgs.hdf5 self.numpy self.numexpr ]; + } + ); + + tensorpack = super.tensorpack.overrideAttrs ( + old: { + postPatch = '' + substituteInPlace setup.cfg --replace "# will call find_packages()" "" + ''; + } + ); + urwidtrees = super.urwidtrees.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ [