Merge pull request #81621 from marsam/update-scrapy

pythonPackages.scrapy: 1.8.0 -> 2.0.0

pkgs/development/python-modules/protego/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, six
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "Protego";
+  version = "0.1.16";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4";
+  };
+  propagatedBuildInputs = [ six ];
+
+  checkInputs = [ pytest ];
+
+  checkPhase = ''
+    pytest tests
+  '';
+
+  meta = with lib; {
+    description = "A pure-Python robots.txt parser with support for modern conventions";
+    homepage = "https://github.com/scrapy/protego";
+    license = licenses.bsd3;
+    maintainers = [ maintainers.marsam ];
+  };
+}

pkgs/development/python-modules/pytest-twisted/default.nix

@@ -0,0 +1,27 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, greenlet
+, pytest
+, decorator
+}:
+
+buildPythonPackage rec {
+  pname = "pytest-twisted";
+  version = "1.12";
+
+  src = fetchPypi {
+    inherit pname version;
+    extension = "zip";
+    sha256 = "bb9af117c5c6063d9ef20ffdf2fa297caaf57de5a687e4d3607db7b0a6f74fea";
+  };
+
+  propagatedBuildInputs = [ greenlet pytest decorator ];
+
+  meta = with lib; {
+    description = "A twisted plugin for py.test";
+    homepage = "https://github.com/pytest-dev/pytest-twisted";
+    license = licenses.bsd3;
+    maintainers = [ maintainers.marsam ];
+  };
+}

pkgs/development/python-modules/scrapy/default.nix

@@ -1,13 +1,60 @@
-{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
-  testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
-  service-identity, parsel, pydispatcher, cssselect, lib }:
+{ stdenv
+, buildPythonPackage
+, isPy27
+, fetchPypi
+, glibcLocales
+, pytest
+, testfixtures
+, pillow
+, twisted
+, cryptography
+, w3lib
+, lxml
+, queuelib
+, pyopenssl
+, service-identity
+, parsel
+, pydispatcher
+, cssselect
+, zope_interface
+, protego
+, lib
+, jmespath
+, sybil
+, pytest-twisted
+, botocore
+}:
+
 buildPythonPackage rec {
-  version = "1.8.0";
+  version = "2.0.1";
   pname = "Scrapy";
 
-  checkInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
+  disabled = isPy27;
+
+  checkInputs = [
+    glibcLocales
+    jmespath
+    pytest
+    sybil
+    testfixtures
+    pillow
+    pytest-twisted
+    botocore
+  ];
+
   propagatedBuildInputs = [
-    six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
+    twisted
+    cryptography
+    cssselect
+    lxml
+    parsel
+    pydispatcher
+    pyopenssl
+    queuelib
+    service-identity
+    w3lib
+    zope_interface
+    protego
   ];
 
   patches = [
@@ -18,20 +65,20 @@ buildPythonPackage rec {
     ./permissions-fix.patch
   ];
 
-  LC_ALL="en_US.UTF-8";
+  LC_ALL = "en_US.UTF-8";
 
   # Disable doctest plugin—enabled in the shipped pytest.ini—because it causes pytest to hang
   # Ignore proxy tests because requires mitmproxy
   # Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
   # Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
   checkPhase = ''
-    substituteInPlace pytest.ini --replace "addopts = --doctest-modules" "addopts ="
+    substituteInPlace pytest.ini --replace "--doctest-modules" ""
     pytest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
   '';
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "fe06576f9a4971de9dc0175c60fd92561e8275f2bad585c1cb5d65c5181b2db0";
+    sha256 = "85581a01f4160a103ca9906ffa4e44474f4ecd1685f0934728892c58ebf111f6";
   };
 
   postInstall = ''
@@ -42,7 +89,7 @@ buildPythonPackage rec {
 
   meta = with lib; {
     description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
-    homepage = https://scrapy.org/;
+    homepage = "https://scrapy.org/";
     license = licenses.bsd3;
     maintainers = with maintainers; [ drewkett marsam ];
     platforms = platforms.unix;

pkgs/development/python-modules/sybil/default.nix

@@ -1,31 +1,28 @@
-{ stdenv, buildPythonApplication, fetchPypi, fetchpatch
-, pytest, nose }:
+{ lib
+, buildPythonApplication
+, fetchPypi
+, pytest
+, nose
+}:
 
 buildPythonApplication rec {
-  pname   = "sybil";
-  version = "1.0.9";
+  pname = "sybil";
+  version = "1.2.2";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "41d2f1dba8fd1d8ead5e9b1220b590fab8b0d1ca01d43da08555b1fb08d4d8e8";
+    sha256 = "dd84e68facfcb778298ef50a4d7446d4d9092e9d8596012b12bcb82858fd10e1";
   };
 
-  patches = [
-    (fetchpatch {
-      url = https://github.com/cjw296/sybil/commit/6461d8156cfb68bd073ec613a5a516916e97e549.patch;
-      sha256 = "0aqny0i7l6g6d7vr025b90zz8wzszqdbmi05mp67dxw5xqjqvxj2";
-    })
-  ];
-
   checkInputs = [ pytest nose ];
 
   checkPhase = ''
     py.test tests
   '';
 
-  meta = with stdenv.lib; {
-    description = "Automated testing for the examples in your documentation.";
-    homepage    = https://github.com/cjw296/sybil/;
-    license     = licenses.mit;
+  meta = with lib; {
+    description = "Automated testing for the examples in your documentation";
+    homepage = "https://github.com/cjw296/sybil";
+    license = licenses.mit;
   };
 }

pkgs/top-level/python-packages.nix

@@ -1292,6 +1292,8 @@ in {
 
   pytest-tornado = callPackage ../development/python-modules/pytest-tornado { };
 
+  pytest-twisted = callPackage ../development/python-modules/pytest-twisted { };
+
   pytest-xprocess = callPackage ../development/python-modules/pytest-xprocess { };
 
   pytest-xvfb = callPackage ../development/python-modules/pytest-xvfb { };
@@ -5198,6 +5200,8 @@ in {
 
   progressbar33 = callPackage ../development/python-modules/progressbar33 { };
 
+  protego = callPackage ../development/python-modules/protego {};
+
   ldap = callPackage ../development/python-modules/ldap {
     inherit (pkgs) openldap cyrus_sasl;
   };