From cca0b0117b1935959d844d40eefd3522a9d1c331 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Sat, 16 Jan 2021 15:07:33 +0100
Subject: [PATCH] python3Packages.tokenizers: 0.9.4 -> 0.10.0

Changelog:
https://github.com/huggingface/tokenizers/releases/tag/python-v0.10.0
---
 .../python-modules/tokenizers/default.nix       | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/python-modules/tokenizers/default.nix b/pkgs/development/python-modules/tokenizers/default.nix
index d5d92b19061a..ec435a0ac359 100644
--- a/pkgs/development/python-modules/tokenizers/default.nix
+++ b/pkgs/development/python-modules/tokenizers/default.nix
@@ -7,6 +7,7 @@
 , wheel
 , numpy
 , python
+, datasets
 , pytestCheckHook
 , requests
 }:
@@ -50,16 +51,16 @@ let
   };
 in rustPlatform.buildRustPackage rec {
   pname = "tokenizers";
-  version = "0.9.4";
+  version = "0.10.0";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "python-v${version}";
-    hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg=";
+    hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
   };
 
-  cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M=";
+  cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
 
   sourceRoot = "source/bindings/python";
 
@@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
   ];
 
   installCheckInputs = [
+    datasets
     pytestCheckHook
     requests
   ];
@@ -105,6 +107,15 @@ in rustPlatform.buildRustPackage rec {
     pipInstallPhase
   '';
 
+  preCheck = ''
+    HOME=$TMPDIR
+  '';
+
+  disabledTests = [
+    # Downloads data using the datasets module.
+    "TestTrainFromIterators"
+  ];
+
   meta = with lib; {
     homepage = "https://github.com/huggingface/tokenizers";
     description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";