JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

poetry2nix: 1.13.0 -> 1.14.0

Browse Source
This commit is contained in:
adisbladis 2020-11-30 19:55:39 +01:00
parent b1ed5ffeab
commit a10370cc9f
No known key found for this signature in database
GPG Key ID: 110BFAD44C6249B7
10 changed files with 269 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix
View File

@ -7,60 +7,127 @@ import toml
import json
import sys
argparser = argparse.ArgumentParser(description="Poetry2nix CLI")
subparsers = argparser.add_subparsers(dest="subcommand")
subparsers.required = True
parser_lock = subparsers.add_parser("lock", help="Generate overrides for git hashes",)
parser_lock.add_argument(
"--lock", default="poetry.lock", help="Path to input poetry.lock",
)
parser_lock.add_argument(
"--out", default="poetry-git-overlay.nix", help="Output file",
)
from typing import Dict, Any, Tuple, List
def fetch_git(pkg):
return (
pkg["name"],
subprocess.run(
[
"nix-prefetch-git",
"--fetch-submodules",
"--url",
pkg["source"]["url"],
"--rev",
pkg["source"]["reference"],
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
),
class Package:
def __init__(self, attrs: Dict[str, Any]) -> None:
self.attrs = attrs
self.name = attrs["name"]
self.source = self.attrs["source"]
def fetch(self) -> Tuple["Package", subprocess.CompletedProcess]:
raise NotImplementedError()
def expression(self, output: str) -> str:
raise NotImplementedError()
class UrlPackage(Package):
def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
return (
self,
subprocess.run(
[
"nix-prefetch-url",
"--unpack",
self.source["url"],
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
),
)
def expression(self, output: str) -> str:
sha256 = output.rstrip()
return textwrap.dedent("""
%s = super.%s.overridePythonAttrs (
_: {
src = pkgs.fetchzip {
url = "%s";
sha256 = "%s";
};
}
);""" % (self.name, self.name, self.source["url"], sha256))
class GitPackage(Package):
def fetch(self) -> Tuple[Package, subprocess.CompletedProcess]:
reference = self.source.get("resolved_reference", self.source["reference"])
return (
self,
subprocess.run(
[
"nix-prefetch-git",
"--fetch-submodules",
"--url",
self.source["url"],
"--rev",
reference,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
),
)
def expression(self, output: str) -> str:
meta = json.loads(output)
return textwrap.dedent("""
%s = super.%s.overridePythonAttrs (
_: {
src = pkgs.fetchgit {
url = "%s";
rev = "%s";
sha256 = "%s";
};
}
);""" % (self.name, self.name, meta["url"], meta["rev"], meta["sha256"]))
def parse_args() -> argparse.Namespace:
argparser = argparse.ArgumentParser(description="Poetry2nix CLI")
subparsers = argparser.add_subparsers(dest="subcommand")
subparsers.required = True
parser_lock = subparsers.add_parser("lock", help="Generate overrides for git hashes",)
parser_lock.add_argument(
"--lock", default="poetry.lock", help="Path to input poetry.lock",
)
parser_lock.add_argument(
"--out", default="poetry-git-overlay.nix", help="Output file",
)
return argparser.parse_args()
def indent(expr, spaces=2):
def indent(expr: str, spaces: int = 2) -> str:
i = " " * spaces
return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")])
if __name__ == "__main__":
args = argparser.parse_args()
def main() -> None:
args = parse_args()
with open(args.lock) as lockf:
lock = toml.load(lockf)
pkgs = []
pkgs: List[Package] = []
for pkg in lock["package"]:
if "source" in pkg:
pkgs.append(pkg)
source_type = pkg["source"]["type"]
if source_type == "git":
pkgs.append(GitPackage(pkg))
elif source_type == "url":
pkgs.append(UrlPackage(pkg))
with ThreadPoolExecutor() as e:
futures = []
for pkg in pkgs:
futures.append(e.submit(fetch_git, pkg))
futures.append(e.submit(pkg.fetch))
lines = [
"{ pkgs }:",
@ -68,30 +135,13 @@ if __name__ == "__main__":
]
for f in futures:
drv_name, p = f.result()
package, p = f.result()
if p.returncode != 0:
sys.stderr.buffer.write(p.stderr)
sys.stderr.buffer.flush()
sys.stderr.write(p.stderr)
sys.stderr.flush()
exit(p.returncode)
meta = json.loads(p.stdout.decode())
lines.append(
indent(
textwrap.dedent(
"""
%s = super.%s.overridePythonAttrs (
_: {
src = pkgs.fetchgit {
url = "%s";
rev = "%s";
sha256 = "%s";
};
}
);"""
% (drv_name, drv_name, meta["url"], meta["rev"], meta["sha256"])
)
)
)
expr = package.expression(p.stdout)
lines.append(indent(expr))
lines.extend(["", "}", ""])
@ -101,3 +151,7 @@ if __name__ == "__main__":
fout.write(expr)
print(f"Wrote {args.out}")
if __name__ == "__main__":
main()

2
pkgs/development/tools/poetry2nix/poetry2nix/default.nix
View File

@ -71,7 +71,7 @@ in
lib.makeScope pkgs.newScope (self: {
# Poetry2nix version
version = "1.13.0";
version = "1.14.0";
/*
Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.

6
pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix
View File

@ -1,11 +1,11 @@
{ python
, callPackage
, buildPackages
, makeSetupHook
, yj
, wheel
, pip
}:
let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
in
@ -20,7 +20,7 @@ in
deps = [ ];
substitutions = {
inherit pythonInterpreter;
yj = "${yj}/bin/yj";
yj = "${buildPackages.yj}/bin/yj";
pyprojectPatchScript = "${./pyproject-without-path.py}";
};
} ./remove-path-dependencies.sh

27
pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix
View File

@ -27,6 +27,7 @@ pythonPackages.callPackage
, ...
}@args:
let
inherit (pkgs) stdenv;
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName;
inherit (import ./pep425.nix {
@ -45,6 +46,7 @@ pythonPackages.callPackage
toPath = s: pwd + "/${s}";
isSource = source != null;
isGit = isSource && source.type == "git";
isUrl = isSource && source.type == "url";
isLocal = isSource && source.type == "directory";
localDepPath = toPath source.url;
@ -91,7 +93,7 @@ pythonPackages.callPackage
"toml" # Toml is an extra for setuptools-scm
];
baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
format = if isLocal then "pyproject" else if isGit then "pyproject" else fileInfo.format;
format = if isLocal || isGit || isUrl then "pyproject" else fileInfo.format;
in
buildPythonPackage {
pname = moduleName name;
@ -113,9 +115,10 @@ pythonPackages.callPackage
buildInputs = (
baseBuildInputs
++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
++ lib.optional isLocal buildSystemPkgs
++ lib.optional (!__isBootstrap) [ pythonPackages.poetry ]
++ lib.optional (!__isBootstrap) pythonPackages.poetry
);
propagatedBuildInputs =
@ -157,14 +160,22 @@ pythonPackages.callPackage
(
builtins.fetchGit {
inherit (source) url;
rev = source.reference;
rev = source.resolved_reference or source.reference;
ref = sourceSpec.branch or sourceSpec.rev or sourceSpec.tag or "HEAD";
}
) else if isLocal then (poetryLib.cleanPythonSources { src = localDepPath; }) else
fetchFromPypi {
pname = name;
inherit (fileInfo) file hash kind;
};
)
else if isUrl then
builtins.fetchTarball
{
inherit (source) url;
}
else if isLocal then
(poetryLib.cleanPythonSources { src = localDepPath; })
else
fetchFromPypi {
pname = name;
inherit (fileInfo) file hash kind;
};
}
)
{ }

134
pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix
View File

@ -93,7 +93,7 @@ self: super:
(
super.cffi.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ pkgs.libffi ];
buildInputs = old.buildInputs or [ ] ++ [ pkgs.libffi ];
}
)
);
@ -106,6 +106,12 @@ self: super:
}
);
colour = super.colour.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ self.d2to1 ];
}
);
configparser = super.configparser.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [
@ -120,6 +126,8 @@ self: super:
cryptography = super.cryptography.overridePythonAttrs (
old: {
nativeBuildInputs = old.nativeBuildInputs or [ ]
++ stdenv.lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) self.python.pythonForBuild.pkgs.cffi;
buildInputs = old.buildInputs ++ [ pkgs.openssl ];
}
);
@ -324,6 +332,17 @@ self: super:
}
);
jira = super.jira.overridePythonAttrs (
old: {
inherit (pkgs.python3Packages.jira) patches;
buildInputs = old.buildInputs ++ [
self.pytestrunner
self.cryptography
self.pyjwt
];
}
);
jsonpickle = super.jsonpickle.overridePythonAttrs (
old: {
dontPreferSetupPy = true;
@ -499,6 +518,31 @@ self: super:
buildInputs = oa.buildInputs ++ [ self.pbr ];
});
mpi4py = super.mpi4py.overridePythonAttrs (
old:
let
cfg = pkgs.writeTextFile {
name = "mpi.cfg";
text = (
lib.generators.toINI
{ }
{
mpi = {
mpicc = "${pkgs.openmpi.outPath}/bin/mpicc";
};
}
);
};
in
{
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ];
enableParallelBuilding = true;
preBuild = ''
ln -sf ${cfg} mpi.cfg
'';
}
);
multiaddr = super.multiaddr.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ self.pytest-runner ];
@ -584,8 +628,8 @@ self: super:
withMysql = old.passthru.withMysql or false;
in
{
buildInputs = old.buildInputs ++ [ self.cython pkgs.sqlite ];
propagatedBuildInputs = old.propagatedBuildInputs
buildInputs = old.buildInputs or [ ] ++ [ pkgs.sqlite ];
propagatedBuildInputs = old.propagatedBuildInputs or [ ]
++ lib.optional withPostgres self.psycopg2
++ lib.optional withMysql self.mysql-connector;
}
@ -602,8 +646,8 @@ self: super:
# "Vendor" dependencies (for build-system support)
postPatch = ''
echo "import sys" >> poetry/__init__.py
for path in ''${PYTHONPATH//:/ }; do echo $path; done | uniq | while read path; do
echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
for path in $propagatedBuildInputs; do
echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
done
'';
@ -796,6 +840,14 @@ self: super:
}
);
python-bugzilla = super.python-bugzilla.overridePythonAttrs (
old: {
nativeBuildInputs = old.nativeBuildInputs ++ [
self.docutils
];
}
);
python-ldap = super.python-ldap.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ pkgs.openldap pkgs.cyrus_sasl ];
@ -924,6 +976,15 @@ self: super:
}
);
pytest-django = super.pytest-django.overridePythonAttrs (
old: {
postPatch = ''
substituteInPlace setup.py --replace "'pytest>=3.6'," ""
substituteInPlace setup.py --replace "'pytest>=3.6'" ""
'';
}
);
pytest-runner = super.pytest-runner or super.pytestrunner;
python-jose = super.python-jose.overridePythonAttrs (
@ -1090,6 +1151,43 @@ self: super:
}
);
torch = lib.makeOverridable
({ enableCuda ? false
, cudatoolkit ? pkgs.cudatoolkit_10_1
, pkg ? super.torch
}: pkg.overrideAttrs (old:
{
preConfigure =
if (!enableCuda) then ''
export USE_CUDA=0
'' else ''
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${cudatoolkit}/targets/x86_64-linux/lib"
'';
preFixup = lib.optionalString (!enableCuda) ''
# For some reason pytorch retains a reference to libcuda even if it
# is explicitly disabled with USE_CUDA=0.
find $out -name "*.so" -exec ${pkgs.patchelf}/bin/patchelf --remove-needed libcuda.so.1 {} \;
'';
buildInputs = old.buildInputs ++ lib.optionals enableCuda [
pkgs.linuxPackages.nvidia_x11
pkgs.nccl.dev
pkgs.nccl.out
];
propagatedBuildInputs = [
super.numpy
super.future
];
})
)
{ };
typeguard = super.typeguard.overridePythonAttrs (old: {
postPatch = ''
substituteInPlace setup.py \
--replace 'setup()' 'setup(version="${old.version}")'
'';
});
# nix uses a dash, poetry uses an underscore
typing_extensions = super.typing_extensions or self.typing-extensions;
@ -1193,4 +1291,30 @@ self: super:
}
);
credis = super.credis.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ self.cython ];
}
);
hashids = super.hashids.overridePythonAttrs (
old: {
buildInputs = old.buildInputs ++ [ self.flit-core ];
}
);
supervisor = super.supervisor.overridePythonAttrs (
old: {
propagatedBuildInputs = old.propagatedBuildInputs ++ [
self.meld3
self.setuptools
];
}
);
cytoolz = super.cytoolz.overridePythonAttrs (
old: {
propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.toolz ];
}
);
}

2
pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix
View File

@ -87,7 +87,7 @@ let
filtered = builtins.filter filterWheel filesWithoutSources;
choose = files:
let
osxMatches = [ "10_12" "10_11" "10_10" "10_9" "any" ];
osxMatches = [ "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ];
chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x);
chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);

4
pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/default.nix
View File

@ -15,8 +15,8 @@ poetry2nix.mkPoetryApplication {
# "Vendor" dependencies (for build-system support)
postPatch = ''
echo "import sys" >> poetry/__init__.py
for path in ''${PYTHONPATH//:/ }; do echo $path; done | uniq | while read path; do
echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
for path in $propagatedBuildInputs; do
echo "sys.path.insert(0, \"$path\")" >> poetry/__init__.py
done
'';

2
pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "poetry"
version = "1.1.0"
version = "1.1.4"
description = "Python dependency management and packaging made easy."
authors = [
"Sébastien Eustace <sebastien@eustace.io>"

4
pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/src.json
View File

@ -1,7 +1,7 @@
{
"owner": "python-poetry",
"repo": "poetry",
"rev": "539d7f732c34c821258a9853cd3078cbda34a717",
"sha256": "0kl23dkq9n112z1pqjg6f1wv3qk77ij6q5glg15lwrj7yrl9k65c",
"rev": "8312e3f2dbfa126cd311c666fea30656941e1bd3",
"sha256": "0lx3qpz5dad0is7ki5a4vxphvc8cm8fnv4bmrx226a6nvvaj6ahs",
"fetchSubmodules": true
}

1
pkgs/development/tools/poetry2nix/poetry2nix/plugins.nix
View File

@ -23,6 +23,7 @@ let
dontConfigure = true;
dontBuild = true;
dontUsePythonRecompileBytecode = true;
passthru = {
inherit (drv.passthru) withPlugins;