JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #109532 from danieldk/tokenizers-0.10.0

Browse Source 
python3Packages.tokenizers: 0.9.4 -> 0.10.0
This commit is contained in:
Guillaume Girol 2021-01-23 16:56:24 +00:00 committed by GitHub
commit 4f33765575
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkgs/development/python-modules/tokenizers/default.nix
View File

@ -7,6 +7,7 @@
, wheel
, numpy
, python
, datasets
, pytestCheckHook
, requests
}:
@ -50,16 +51,16 @@ let
};
in rustPlatform.buildRustPackage rec {
pname = "tokenizers";
version = "0.9.4";
version = "0.10.0";
src = fetchFromGitHub {
owner = "huggingface";
repo = pname;
rev = "python-v${version}";
hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg=";
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
};
cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M=";
cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
sourceRoot = "source/bindings/python";
@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
];
installCheckInputs = [
datasets
pytestCheckHook
requests
];
@ -105,6 +107,15 @@ in rustPlatform.buildRustPackage rec {
pipInstallPhase
'';
preCheck = ''
HOME=$TMPDIR
'';
disabledTests = [
# Downloads data using the datasets module.
"TestTrainFromIterators"
];
meta = with lib; {
homepage = "https://github.com/huggingface/tokenizers";
description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";