2021-01-10 07:08:02 +00:00
|
|
|
{ lib
|
|
|
|
, stdenv
|
2020-03-03 09:22:00 +00:00
|
|
|
, buildPythonPackage
|
|
|
|
, isPy27
|
|
|
|
, fetchPypi
|
|
|
|
, glibcLocales
|
2020-10-11 23:33:00 +01:00
|
|
|
, pytestCheckHook
|
2020-03-03 09:22:00 +00:00
|
|
|
, testfixtures
|
|
|
|
, pillow
|
|
|
|
, twisted
|
|
|
|
, cryptography
|
|
|
|
, w3lib
|
|
|
|
, lxml
|
|
|
|
, queuelib
|
|
|
|
, pyopenssl
|
|
|
|
, service-identity
|
|
|
|
, parsel
|
|
|
|
, pydispatcher
|
|
|
|
, cssselect
|
|
|
|
, zope_interface
|
|
|
|
, protego
|
|
|
|
, jmespath
|
|
|
|
, sybil
|
|
|
|
, pytest-twisted
|
|
|
|
, botocore
|
2020-07-01 10:21:00 +01:00
|
|
|
, itemadapter
|
2020-08-05 02:21:20 +01:00
|
|
|
, itemloaders
|
2020-03-03 09:22:00 +00:00
|
|
|
}:
|
|
|
|
|
2017-02-15 22:01:38 +00:00
|
|
|
buildPythonPackage rec {
|
2020-11-23 23:00:00 +00:00
|
|
|
version = "2.4.1";
|
2018-06-23 14:27:58 +01:00
|
|
|
pname = "Scrapy";
|
2017-02-15 22:01:38 +00:00
|
|
|
|
2020-03-03 09:22:00 +00:00
|
|
|
disabled = isPy27;
|
|
|
|
|
|
|
|
checkInputs = [
|
|
|
|
glibcLocales
|
|
|
|
jmespath
|
2020-10-11 23:33:00 +01:00
|
|
|
pytestCheckHook
|
2020-03-03 09:22:00 +00:00
|
|
|
sybil
|
|
|
|
testfixtures
|
|
|
|
pillow
|
|
|
|
pytest-twisted
|
|
|
|
botocore
|
|
|
|
];
|
|
|
|
|
2018-06-23 14:27:58 +01:00
|
|
|
propagatedBuildInputs = [
|
2020-03-03 09:22:00 +00:00
|
|
|
twisted
|
|
|
|
cryptography
|
|
|
|
cssselect
|
|
|
|
lxml
|
|
|
|
parsel
|
|
|
|
pydispatcher
|
|
|
|
pyopenssl
|
|
|
|
queuelib
|
|
|
|
service-identity
|
|
|
|
w3lib
|
|
|
|
zope_interface
|
|
|
|
protego
|
2020-07-01 10:21:00 +01:00
|
|
|
itemadapter
|
2020-08-05 02:21:20 +01:00
|
|
|
itemloaders
|
2018-12-23 03:21:54 +00:00
|
|
|
];
|
2017-02-15 22:01:38 +00:00
|
|
|
|
2020-03-03 09:22:00 +00:00
|
|
|
LC_ALL = "en_US.UTF-8";
|
2017-02-15 22:01:38 +00:00
|
|
|
|
2020-10-11 23:33:00 +01:00
|
|
|
# Disable doctest plugin because it causes pytest to hang
|
|
|
|
preCheck = ''
|
2020-03-03 09:22:00 +00:00
|
|
|
substituteInPlace pytest.ini --replace "--doctest-modules" ""
|
2018-06-23 14:27:58 +01:00
|
|
|
'';
|
2017-02-15 22:01:38 +00:00
|
|
|
|
2020-10-11 23:33:00 +01:00
|
|
|
pytestFlagsArray = [
|
|
|
|
"--ignore=tests/test_proxy_connect.py"
|
|
|
|
"--ignore=tests/test_utils_display.py"
|
|
|
|
"--ignore=tests/test_command_check.py"
|
|
|
|
];
|
|
|
|
|
|
|
|
disabledTests = [
|
|
|
|
"FTPFeedStorageTest"
|
|
|
|
"test_noconnect"
|
|
|
|
"test_retry_dns_error"
|
|
|
|
"test_custom_asyncio_loop_enabled_true"
|
|
|
|
"test_custom_loop_asyncio"
|
2021-01-24 00:29:22 +00:00
|
|
|
] ++ lib.optionals stdenv.isDarwin [
|
2021-01-10 07:08:02 +00:00
|
|
|
"test_xmliter_encoding"
|
|
|
|
"test_download"
|
|
|
|
];
|
2020-10-11 23:33:00 +01:00
|
|
|
|
2018-06-23 14:27:58 +01:00
|
|
|
src = fetchPypi {
|
|
|
|
inherit pname version;
|
2020-11-23 23:00:00 +00:00
|
|
|
sha256 = "68c48f01a58636bdf0f6fcd5035a19ecf277b58af24bd70c36dc6e556df3e005";
|
2018-06-23 14:27:58 +01:00
|
|
|
};
|
2017-02-15 22:01:38 +00:00
|
|
|
|
2018-12-23 03:21:54 +00:00
|
|
|
postInstall = ''
|
|
|
|
install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
|
|
|
|
install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
|
|
|
|
install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
|
|
|
|
'';
|
|
|
|
|
2021-01-10 07:08:02 +00:00
|
|
|
__darwinAllowLocalNetworking = true;
|
|
|
|
|
2021-01-11 07:54:33 +00:00
|
|
|
meta = with lib; {
|
2018-06-23 14:27:58 +01:00
|
|
|
description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
|
2020-03-03 09:22:00 +00:00
|
|
|
homepage = "https://scrapy.org/";
|
2018-06-23 14:27:58 +01:00
|
|
|
license = licenses.bsd3;
|
2018-12-23 03:21:54 +00:00
|
|
|
maintainers = with maintainers; [ drewkett marsam ];
|
2018-06-23 14:27:58 +01:00
|
|
|
platforms = platforms.unix;
|
|
|
|
};
|
2017-02-15 22:01:38 +00:00
|
|
|
}
|