JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #52719 from marsam/fix-scrapy-python37

Browse Source 
pythonPackages.scrapy: fix build on Python 3.7
This commit is contained in:
Maximilian Bosch 2018-12-23 14:53:25 +01:00 committed by GitHub
commit db5399b188
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
pkgs/development/python-modules/scrapy/default.nix
View File

@ -1,6 +1,6 @@
{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
service-identity, parsel, pydispatcher, cssselect, lib }:
service-identity, parsel, pydispatcher, cssselect, lib, fetchpatch }:
buildPythonPackage rec {
version = "1.5.1";
pname = "Scrapy";
@ -10,18 +10,26 @@ buildPythonPackage rec {
six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
];
# Scrapy is usually installed via pip where copying all
# permissions makes sense. In Nix the files copied are owned by
# root and readonly. As a consequence scrapy can't edit the
# project templates.
patches = [ ./permissions-fix.patch ];
patches = [
# Scrapy is usually installed via pip where copying all
# permissions makes sense. In Nix the files copied are owned by
# root and readonly. As a consequence scrapy can't edit the
# project templates.
./permissions-fix.patch
# fix python37 issues. Remove with the next release
(fetchpatch {
url = https://github.com/scrapy/scrapy/commit/f4f39057cbbfa4daf66f82061e57101b88d88d05.patch;
sha256 = "1f761qkji362i20i5bzcxz44sihvl29prm02i5l2xyhgl1hp91hv";
})
];
LC_ALL="en_US.UTF-8";
# Ignore proxy tests because requires mitmproxy
# Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
# Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
checkPhase = ''
py.test --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py ${lib.optionalString stdenv.isDarwin "--ignore=tests/test_utils_iterators.py"}
# The ignored tests require mitmproxy, which depends on protobuf, but it's disabled on Python3
# Ignore iteration test, because lxml can't find encodings on darwin https://bugs.launchpad.net/lxml/+bug/707396
pytest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
'';
src = fetchPypi {
@ -29,11 +37,17 @@ buildPythonPackage rec {
sha256 = "5a398bf6818f87dcc817c919408a195f19ba46414ae12f259119336cfa862bb6";
};
postInstall = ''
install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
'';
meta = with lib; {
description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
homepage = https://scrapy.org/;
license = licenses.bsd3;
maintainers = with maintainers; [ drewkett ];
maintainers = with maintainers; [ drewkett marsam ];
platforms = platforms.unix;
};
}