Merge pull request #44657 from costrouc/costrouc/dask-distributed

pythonPackages.distributed: 1.15.1 -> 1.22.1

pkgs/development/python-modules/distributed/default.nix

@@ -0,0 +1,62 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytest
+, pytest-repeat
+, pytest-faulthandler
+, pytest-timeout
+, mock
+, joblib
+, click
+, cloudpickle
+, dask
+, msgpack
+, psutil
+, six
+, sortedcontainers
+, tblib
+, toolz
+, tornado
+, zict
+, pyyaml
+, pythonOlder
+, futures
+, singledispatch
+}:
+
+buildPythonPackage rec {
+  pname = "distributed";
+  version = "1.22.1";
+
+  # get full repository need conftest.py to run tests
+  src = fetchFromGitHub {
+    owner = "dask";
+    repo = pname;
+    rev = version;
+    sha256 = "0xvx55rhbhlyys3kjndihwq6y6260qzy9mr3miclh5qddaiw2d5z";
+  };
+
+  checkInputs = [ pytest pytest-repeat pytest-faulthandler pytest-timeout mock joblib ];
+  propagatedBuildInputs = [
+      click cloudpickle dask msgpack psutil six
+      sortedcontainers tblib toolz tornado zict pyyaml
+  ] ++ lib.optional (pythonOlder "3.2") [ futures ]
+    ++ lib.optional (pythonOlder "3.4") [ singledispatch ];
+
+  # tests take about 10-15 minutes
+  # ignore 5 cli tests out of 1000 total tests that fail due to subprocesses
+  # these tests are not critical to the library (only the cli)
+  checkPhase = ''
+    py.test distributed -m "not avoid-travis" -r s --timeout-method=thread --timeout=0 --durations=20 --ignore="distributed/cli/tests"
+  '';
+
+  # when tested random tests would fail and not repeatably
+  doCheck = false;
+
+  meta = {
+    description = "Distributed computation in Python.";
+    homepage = http://distributed.readthedocs.io/en/latest/;
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ teh costrouc ];
+  };
+}

pkgs/development/python-modules/joblib/default.nix

@@ -1,31 +1,36 @@
 { lib
 , buildPythonPackage
-, fetchPypi
+, fetchFromGitHub
 , sphinx
 , numpydoc
 , pytest
+, python-lz4
 }:
 
 
 buildPythonPackage rec {
   pname = "joblib";
-  version = "0.12.3";
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "333b9bf16ff015d6b56bf80b9831afdd243443cb84c7ff7b6e342f117e354c42";
+  version = "0.12.4";
+
+  # get full repository inorder to run tests
+  src = fetchFromGitHub {
+    owner = "joblib";
+    repo = pname;
+    rev = version;
+    sha256 = "06zszgp7wpa4jr554wkk6kkigp4k9n5ad5h08i6w9qih963rlimb";
   };
 
   checkInputs = [ sphinx numpydoc pytest ];
+  propagatedBuildInputs = [ python-lz4 ];
 
   checkPhase = ''
-    py.test -k 'not test_disk_used and \
-                not test_nested_parallel_warnings and \
-                not test_nested_parallelism_limit' joblib/test
+    py.test joblib
   '';
 
   meta = {
     description = "Lightweight pipelining: using Python functions as pipeline jobs";
     homepage = https://pythonhosted.org/joblib/;
     license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ costrouc ];
   };
 }

pkgs/development/python-modules/pytest-faulthandler/default.nix

@@ -0,0 +1,34 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, setuptools_scm
+, pytest
+, pytest-mock
+, pythonOlder
+, faulthandler
+}:
+
+buildPythonPackage rec {
+  pname = "pytest-faulthandler";
+  version = "1.5.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "bf8634c3fd6309ef786ec03b913a5366163fdb094ebcfdebc35626400d790e0d";
+  };
+
+  buildInputs = [ setuptools_scm pytest ];
+  checkInputs = [ pytest-mock ];
+  propagatedBuildInputs = lib.optional (pythonOlder "3.0") faulthandler;
+
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = {
+    description = "Py.test plugin that activates the fault handler module for tests";
+    homepage = https://github.com/pytest-dev/pytest-faulthandler;
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ costrouc ];
+  };
+}

pkgs/development/python-modules/pytest-repeat/default.nix

@@ -0,0 +1,29 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, setuptools_scm
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "pytest-repeat";
+  version = "0.6.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "84aba2fcca5dc2f32ae626a01708f469f17b3384ec3d1f507698077f274909d6";
+  };
+
+  buildInputs = [ setuptools_scm pytest ];
+
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = {
+    description = "Pytest plugin for repeating tests";
+    homepage = https://github.com/pytest-dev/pytest-repeat;
+    maintainers = with lib.maintainers; [ costrouc ];
+    license = lib.licenses.mpl20;
+  };
+}

pkgs/development/python-modules/pytest-timeout/default.nix

@@ -1,5 +1,6 @@
 { buildPythonPackage
 , fetchPypi
+, fetchpatch
 , lib
 , pexpect
 , pytest
@@ -13,7 +14,7 @@ buildPythonPackage rec {
     inherit pname version;
     sha256 = "1117fc0536e1638862917efbdc0895e6b62fa61e6cf4f39bb655686af7af9627";
   };
-  buildInputs = [ pytest ];
+
   checkInputs = [ pytest pexpect ];
   checkPhase = ''pytest -ra'';
 
@@ -21,6 +22,6 @@ buildPythonPackage rec {
     description = "py.test plugin to abort hanging tests";
     homepage = https://bitbucket.org/pytest-dev/pytest-timeout/;
     license = licenses.mit;
-    maintainers = with maintainers; [ makefu ];
+    maintainers = with maintainers; [ makefu costrouc ];
   };
 }

pkgs/development/python-modules/python-lz4/default.nix

@@ -0,0 +1,40 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytestrunner
+, pytest
+, psutil
+, setuptools_scm
+, pkgconfig
+, isPy3k
+, future
+}:
+
+buildPythonPackage rec {
+  pname = "python-lz4";
+  version = "2.1.0";
+
+  # get full repository inorder to run tests
+  src = fetchFromGitHub {
+    owner = pname;
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "1vjfplj37jcw1mf8l810dv76dx0raia3ylgyfy7sfsb3g17brjq6";
+  };
+
+  buildInputs = [ setuptools_scm pkgconfig pytestrunner ];
+  checkInputs = [ pytest psutil ];
+  propagatedBuildInputs = lib.optionals (!isPy3k) [ future ];
+
+  # give a hint to setuptools_scm on package version
+  preBuild = ''
+    export SETUPTOOLS_SCM_PRETEND_VERSION="v${version}"
+  '';
+
+  meta = {
+     description = "LZ4 Bindings for Python";
+     homepage = https://github.com/python-lz4/python-lz4;
+     license = lib.licenses.bsd0;
+     maintainers = with lib.maintainers; [ costrouc ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -288,6 +288,8 @@ in {
 
   distorm3 = callPackage ../development/python-modules/distorm3 { };
 
+  distributed = callPackage ../development/python-modules/distributed { };
+
   docutils = callPackage ../development/python-modules/docutils { };
 
   dogtail = callPackage ../development/python-modules/dogtail { };
@@ -501,6 +503,8 @@ in {
 
   python-hosts = callPackage ../development/python-modules/python-hosts { };
 
+  python-lz4 = callPackage ../development/python-modules/python-lz4 { };
+
   python-igraph = callPackage ../development/python-modules/python-igraph {
     pkgconfig = pkgs.pkgconfig;
     igraph = pkgs.igraph;
@@ -1879,6 +1883,8 @@ in {
 
   pytest-django = callPackage ../development/python-modules/pytest-django { };
 
+  pytest-faulthandler = callPackage ../development/python-modules/pytest-faulthandler { };
+
   pytest-fixture-config = callPackage ../development/python-modules/pytest-fixture-config { };
 
   pytest-forked = callPackage ../development/python-modules/pytest-forked { };
@@ -1903,6 +1909,8 @@ in {
 
   pytest-raisesregexp = callPackage ../development/python-modules/pytest-raisesregexp { };
 
+  pytest-repeat = callPackage ../development/python-modules/pytest-repeat { };
+
   pytestrunner = callPackage ../development/python-modules/pytestrunner { };
 
   pytestquickcheck = callPackage ../development/python-modules/pytest-quickcheck { };
@@ -1970,34 +1978,6 @@ in {
 
   zict = callPackage ../development/python-modules/zict { };
 
-  distributed = buildPythonPackage rec {
-
-    name = "distributed-${version}";
-    version = "1.15.1";
-
-    src = pkgs.fetchurl {
-      url = "mirror://pypi/d/distributed/${name}.tar.gz";
-      sha256 = "037a07sdf2ch1d360nqwqz3b4ld8msydng7mw4i5s902v7xr05l6";
-    };
-
-    buildInputs = with self; [ pytest docutils ];
-    propagatedBuildInputs = with self; [
-      dask six boto3 s3fs tblib locket msgpack-python click cloudpickle tornado
-      psutil botocore zict lz4 sortedcollections sortedcontainers
-    ] ++ (if !isPy3k then [ singledispatch ] else []);
-
-    # py.test not picking up local config file, even when running
-    # manually: E ValueError: no option named '--runslow'
-    doCheck = false;
-
-    meta = {
-      description = "Distributed computation in Python.";
-      homepage = "http://distributed.readthedocs.io/en/latest/";
-      license = licenses.bsd3;
-      maintainers = with maintainers; [ teh ];
-    };
-  };
-
   digital-ocean = callPackage ../development/python-modules/digitalocean { };
 
   leather = callPackage ../development/python-modules/leather { };