cde89e3ecf
Pyspark switched to pinning py4j==0.10.9 with v3.0.0 - see this commit: https://github.com/apache/spark/\ commit/fc4e56a54c15e20baf085e6061d3d83f5ce1185d This meant that since the bump to pyspark v3.0.0 - in this commit: https://github.com/NixOS/nixpkgs/\ commit/5181547ae6624b462919a806c4d0888e6e4630f4 - the patch was no longer matching on the 'py4j==0.10.7' string that was working previously. The failing patch went unnoticed previously because the version of py4j pinned by pyspark>=3.0.0 was the same as the py4j provided by nixpkgs. However, a recent PR (#101636) bumped the version of py4j to 0.10.9.1 in this commit: https://github.com/NixOS/nixpkgs/\ commit/43a91282d66223c5cb978d53fbe1033f56dd7f2b which caused the version pinned by pyspark to no longer match the version provided by nixpkgs. FWIW, @jonringer flagged this issue on another PR that tried to bump py4j: #100623. My solution here was to upgrade the patch's target string to match the version found in pyspark's current setup.py.
31 lines
749 B
Nix
31 lines
749 B
Nix
{ buildPythonPackage, fetchPypi, stdenv, py4j }:
|
|
|
|
buildPythonPackage rec {
|
|
pname = "pyspark";
|
|
version = "3.0.1";
|
|
|
|
src = fetchPypi {
|
|
inherit pname version;
|
|
sha256 = "38b485d3634a86c9a2923c39c8f08f003fdd0e0a3d7f07114b2fb4392ce60479";
|
|
};
|
|
|
|
# pypandoc is broken with pandoc2, so we just lose docs.
|
|
postPatch = ''
|
|
sed -i "s/'pypandoc'//" setup.py
|
|
|
|
substituteInPlace setup.py --replace py4j==0.10.9 'py4j>=0.10.9,<0.11'
|
|
'';
|
|
|
|
propagatedBuildInputs = [ py4j ];
|
|
|
|
# Tests assume running spark...
|
|
doCheck = false;
|
|
|
|
meta = with stdenv.lib; {
|
|
description = "Apache Spark";
|
|
homepage = "https://github.com/apache/spark/tree/master/python";
|
|
license = licenses.asl20;
|
|
maintainers = [ maintainers.shlevy ];
|
|
};
|
|
}
|