Merge pull request #81621 from marsam/update-scrapy

pythonPackages.scrapy: 1.8.0 -> 2.0.0
This commit is contained in:
Mario Rodas 2020-03-25 07:22:43 -05:00 committed by GitHub
commit 5ce4be4536
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 131 additions and 26 deletions

View File

@ -0,0 +1,30 @@
{ lib
, buildPythonPackage
, fetchPypi
, six
, pytest
}:
buildPythonPackage rec {
pname = "Protego";
version = "0.1.16";
src = fetchPypi {
inherit pname version;
sha256 = "a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4";
};
propagatedBuildInputs = [ six ];
checkInputs = [ pytest ];
checkPhase = ''
pytest tests
'';
meta = with lib; {
description = "A pure-Python robots.txt parser with support for modern conventions";
homepage = "https://github.com/scrapy/protego";
license = licenses.bsd3;
maintainers = [ maintainers.marsam ];
};
}

View File

@ -0,0 +1,27 @@
{ lib
, buildPythonPackage
, fetchPypi
, greenlet
, pytest
, decorator
}:
buildPythonPackage rec {
pname = "pytest-twisted";
version = "1.12";
src = fetchPypi {
inherit pname version;
extension = "zip";
sha256 = "bb9af117c5c6063d9ef20ffdf2fa297caaf57de5a687e4d3607db7b0a6f74fea";
};
propagatedBuildInputs = [ greenlet pytest decorator ];
meta = with lib; {
description = "A twisted plugin for py.test";
homepage = "https://github.com/pytest-dev/pytest-twisted";
license = licenses.bsd3;
maintainers = [ maintainers.marsam ];
};
}

View File

@ -1,13 +1,60 @@
{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
service-identity, parsel, pydispatcher, cssselect, lib }:
{ stdenv
, buildPythonPackage
, isPy27
, fetchPypi
, glibcLocales
, pytest
, testfixtures
, pillow
, twisted
, cryptography
, w3lib
, lxml
, queuelib
, pyopenssl
, service-identity
, parsel
, pydispatcher
, cssselect
, zope_interface
, protego
, lib
, jmespath
, sybil
, pytest-twisted
, botocore
}:
buildPythonPackage rec {
version = "1.8.0";
version = "2.0.1";
pname = "Scrapy";
checkInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
disabled = isPy27;
checkInputs = [
glibcLocales
jmespath
pytest
sybil
testfixtures
pillow
pytest-twisted
botocore
];
propagatedBuildInputs = [
six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
twisted
cryptography
cssselect
lxml
parsel
pydispatcher
pyopenssl
queuelib
service-identity
w3lib
zope_interface
protego
];
patches = [
@ -18,20 +65,20 @@ buildPythonPackage rec {
./permissions-fix.patch
];
LC_ALL="en_US.UTF-8";
LC_ALL = "en_US.UTF-8";
# Disable doctest plugin—enabled in the shipped pytest.ini—because it causes pytest to hang
# Ignore proxy tests because requires mitmproxy
# Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
# Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
checkPhase = ''
substituteInPlace pytest.ini --replace "addopts = --doctest-modules" "addopts ="
substituteInPlace pytest.ini --replace "--doctest-modules" ""
pytest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
'';
src = fetchPypi {
inherit pname version;
sha256 = "fe06576f9a4971de9dc0175c60fd92561e8275f2bad585c1cb5d65c5181b2db0";
sha256 = "85581a01f4160a103ca9906ffa4e44474f4ecd1685f0934728892c58ebf111f6";
};
postInstall = ''
@ -42,7 +89,7 @@ buildPythonPackage rec {
meta = with lib; {
description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
homepage = https://scrapy.org/;
homepage = "https://scrapy.org/";
license = licenses.bsd3;
maintainers = with maintainers; [ drewkett marsam ];
platforms = platforms.unix;

View File

@ -1,31 +1,28 @@
{ stdenv, buildPythonApplication, fetchPypi, fetchpatch
, pytest, nose }:
{ lib
, buildPythonApplication
, fetchPypi
, pytest
, nose
}:
buildPythonApplication rec {
pname = "sybil";
version = "1.0.9";
pname = "sybil";
version = "1.2.2";
src = fetchPypi {
inherit pname version;
sha256 = "41d2f1dba8fd1d8ead5e9b1220b590fab8b0d1ca01d43da08555b1fb08d4d8e8";
sha256 = "dd84e68facfcb778298ef50a4d7446d4d9092e9d8596012b12bcb82858fd10e1";
};
patches = [
(fetchpatch {
url = https://github.com/cjw296/sybil/commit/6461d8156cfb68bd073ec613a5a516916e97e549.patch;
sha256 = "0aqny0i7l6g6d7vr025b90zz8wzszqdbmi05mp67dxw5xqjqvxj2";
})
];
checkInputs = [ pytest nose ];
checkPhase = ''
py.test tests
'';
meta = with stdenv.lib; {
description = "Automated testing for the examples in your documentation.";
homepage = https://github.com/cjw296/sybil/;
license = licenses.mit;
meta = with lib; {
description = "Automated testing for the examples in your documentation";
homepage = "https://github.com/cjw296/sybil";
license = licenses.mit;
};
}

View File

@ -1292,6 +1292,8 @@ in {
pytest-tornado = callPackage ../development/python-modules/pytest-tornado { };
pytest-twisted = callPackage ../development/python-modules/pytest-twisted { };
pytest-xprocess = callPackage ../development/python-modules/pytest-xprocess { };
pytest-xvfb = callPackage ../development/python-modules/pytest-xvfb { };
@ -5198,6 +5200,8 @@ in {
progressbar33 = callPackage ../development/python-modules/progressbar33 { };
protego = callPackage ../development/python-modules/protego {};
ldap = callPackage ../development/python-modules/ldap {
inherit (pkgs) openldap cyrus_sasl;
};