From acd32a4caf533ca6202afad07c085450649dbe4a Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Fri, 19 May 2017 14:21:58 +0200
Subject: [PATCH] Python: set DETERMINISTIC_BUILD and PYTHONHASHSEED in
 setupHook

The Python interpreters are patched so they can build .pyc bytecode free
of certain indeterminism.

When building Python packages we currently set

```
compiling python files.
in nix store.
DETERMINISTIC_BUILD=1;
PYTHONHASHSEED = 0;
```

Instead if setting these environment variables in the function that
builds the package, this commit sets the variables instead in the Python
setup hook. That way, whenever Python is included in a derivation, these
variables are set.

See also the issue https://github.com/NixOS/nixpkgs/issues/25707.
---
 .../python/cpython/2.7/default.nix            |  3 ++-
 .../python/cpython/2.7/setup-hook.sh          | 15 -------------
 .../python/cpython/3.3/default.nix            |  2 +-
 .../python/cpython/3.3/setup-hook.sh          | 15 -------------
 .../python/cpython/3.4/default.nix            |  3 ++-
 .../python/cpython/3.4/setup-hook.sh          | 15 -------------
 .../python/cpython/3.5/default.nix            |  3 ++-
 .../python/cpython/3.5/setup-hook.sh          | 15 -------------
 .../python/cpython/3.6/default.nix            |  3 ++-
 .../python/cpython/3.6/setup-hook.sh          | 15 -------------
 .../python/mk-python-derivation.nix           |  7 -------
 .../interpreters/python/pypy/2.7/default.nix  |  7 ++++---
 .../python/pypy/2.7/setup-hook.sh             | 15 -------------
 .../interpreters/python/setup-hook.nix        | 13 ++++++++++++
 .../interpreters/python/setup-hook.sh         | 21 +++++++++++++++++++
 pkgs/top-level/all-packages.nix               |  3 +++
 16 files changed, 50 insertions(+), 105 deletions(-)
 delete mode 100644 pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
 delete mode 100644 pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
 delete mode 100644 pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
 delete mode 100644 pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
 delete mode 100644 pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
 delete mode 100644 pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
 create mode 100644 pkgs/development/interpreters/python/setup-hook.nix
 create mode 100644 pkgs/development/interpreters/python/setup-hook.sh

diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 8426902414a7..c5fa05651bbe 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -15,6 +15,7 @@
 , expat
 , libffi
 , CF, configd, coreutils
+, python-setup-hook
 # For the Python package set
 , pkgs, packageOverrides ? (self: super: {})
 }:
@@ -150,7 +151,7 @@ in stdenv.mkDerivation {
     NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2";
     DETERMINISTIC_BUILD = 1;
 
-    setupHook = ./setup-hook.sh;
+    setupHook = python-setup-hook sitePackages;
 
     postPatch = optionalString (x11Support && (tix != null)) ''
           substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
diff --git a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh b/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
deleted file mode 100644
index 4770eea886f6..000000000000
--- a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python2.7/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/lib/python2.7/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix
index c561a1ed750a..061176335c4f 100644
--- a/pkgs/development/interpreters/python/cpython/3.3/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix
@@ -77,7 +77,7 @@ in stdenv.mkDerivation {
                         )
   '';
 
-  setupHook = ./setup-hook.sh;
+  setupHook = python-setup-hook sitePackages;
 
   postInstall = ''
     # needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
deleted file mode 100644
index 82a8c0abd32f..000000000000
--- a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.3/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/lib/python3.3/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix
index b2a4d849c942..4d20a21a4d13 100644
--- a/pkgs/development/interpreters/python/cpython/3.4/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix
@@ -13,6 +13,7 @@
 , callPackage
 , self
 , CF, configd
+, python-setup-hook
 # For the Python package set
 , pkgs, packageOverrides ? (self: super: {})
 }:
@@ -100,7 +101,7 @@ in stdenv.mkDerivation {
      ''}
   '';
 
-  setupHook = ./setup-hook.sh;
+  setupHook = python-setup-hook sitePackages;
 
   postInstall = ''
     # needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
deleted file mode 100644
index fddcc0b73fe8..000000000000
--- a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.4/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/lib/python3.4/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix
index 76f445f7a509..a4aec241a1aa 100644
--- a/pkgs/development/interpreters/python/cpython/3.5/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix
@@ -13,6 +13,7 @@
 , callPackage
 , self
 , CF, configd
+, python-setup-hook
 # For the Python package set
 , pkgs, packageOverrides ? (self: super: {})
 }:
@@ -102,7 +103,7 @@ in stdenv.mkDerivation {
      ''}
   '';
 
-  setupHook = ./setup-hook.sh;
+  setupHook = python-setup-hook sitePackages;
 
   postInstall = ''
     # needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
deleted file mode 100644
index 2836ad7e8f56..000000000000
--- a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.5/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/lib/python3.5/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix
index 150786191669..d2d922ce4955 100644
--- a/pkgs/development/interpreters/python/cpython/3.6/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix
@@ -14,6 +14,7 @@
 , callPackage
 , self
 , CF, configd
+, python-setup-hook
 # For the Python package set
 , pkgs, packageOverrides ? (self: super: {})
 }:
@@ -94,7 +95,7 @@ in stdenv.mkDerivation {
      ''}
   '';
 
-  setupHook = ./setup-hook.sh;
+  setupHook = python-setup-hook sitePackages;
 
   postInstall = ''
     # needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
deleted file mode 100644
index 26a0d57bc87c..000000000000
--- a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.6/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/lib/python3.6/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
index 5d710fcad882..1a388bfe4afa 100644
--- a/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -57,13 +57,6 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
 
   inherit pythonPath;
 
-
-  # Determinism: The interpreter is patched to write null timestamps when compiling python files.
-  # This way python doesn't try to update them when we freeze timestamps in nix store.
-  DETERMINISTIC_BUILD=1;
-  # Determinism: We fix the hashes of str, bytes and datetime objects.
-  PYTHONHASHSEED = 0;
-
   buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath
     ++ [ (ensureNewerSourcesHook { year = "1980"; }) ]
     ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip)
diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix
index 456a078874cd..7552c6cd2853 100644
--- a/pkgs/development/interpreters/python/pypy/2.7/default.nix
+++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix
@@ -1,6 +1,7 @@
 { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi
 , sqlite, openssl, ncurses, python, expat, tcl, tk, tix, xlibsWrapper, libX11
 , makeWrapper, callPackage, self, gdbm, db
+, python-setup-hook
 # For the Python package set
 , pkgs, packageOverrides ? (self: super: {})
 }:
@@ -14,6 +15,7 @@ let
   pythonVersion = "2.7";
   version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
   libPrefix = "pypy${majorVersion}";
+  sitePackages = "site-packages";
 
 in stdenv.mkDerivation rec {
     name = "pypy-${version}";
@@ -67,7 +69,7 @@ in stdenv.mkDerivation rec {
       ${python.interpreter} rpython/bin/rpython --make-jobs="$NIX_BUILD_CORES" -Ojit --batch pypy/goal/targetpypystandalone.py --withmod-_minimal_curses --withmod-unicodedata --withmod-thread --withmod-bz2 --withmod-_multiprocessing
     '';
 
-    setupHook = ./setup-hook.sh;
+    setupHook = python-setup-hook sitePackages;
 
     postBuild = ''
       cd ./lib_pypy
@@ -125,12 +127,11 @@ in stdenv.mkDerivation rec {
     passthru = let
       pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
     in rec {
-      inherit zlibSupport libPrefix;
+      inherit zlibSupport libPrefix sitePackages;
       executable = "pypy";
       isPypy = true;
       buildEnv = callPackage ../../wrapper.nix { python = self; };
       interpreter = "${self}/bin/${executable}";
-      sitePackages = "site-packages";
       withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
       pkgs = pythonPackages;
     };
diff --git a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh b/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
deleted file mode 100644
index e9081d1eaa53..000000000000
--- a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/site-packages
-}
-
-toPythonPath() {
-    local paths="$1"
-    local result=
-    for i in $paths; do
-        p="$i/site-packages"
-        result="${result}${result:+:}$p"
-    done
-    echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/setup-hook.nix b/pkgs/development/interpreters/python/setup-hook.nix
new file mode 100644
index 000000000000..b66bd1cc5f69
--- /dev/null
+++ b/pkgs/development/interpreters/python/setup-hook.nix
@@ -0,0 +1,13 @@
+{ runCommand }:
+
+sitePackages:
+
+let
+  hook = ./setup-hook.sh;
+in runCommand "python-setup-hook.sh" {
+  inherit sitePackages;
+} ''
+  cp ${hook} hook.sh
+  substituteAllInPlace hook.sh
+  mv hook.sh $out
+''
diff --git a/pkgs/development/interpreters/python/setup-hook.sh b/pkgs/development/interpreters/python/setup-hook.sh
new file mode 100644
index 000000000000..dda9bed39f85
--- /dev/null
+++ b/pkgs/development/interpreters/python/setup-hook.sh
@@ -0,0 +1,21 @@
+addPythonPath() {
+    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/@sitePackages@
+}
+
+toPythonPath() {
+    local paths="$1"
+    local result=
+    for i in $paths; do
+        p="$i/@sitePackages@"
+        result="${result}${result:+:}$p"
+    done
+    echo $result
+}
+
+envHooks+=(addPythonPath)
+
+# Determinism: The interpreter is patched to write null timestamps when compiling python files.
+# This way python doesn't try to update them when we freeze timestamps in nix store.
+export DETERMINISTIC_BUILD=1;
+# Determinism: We fix the hashes of str, bytes and datetime objects.
+export PYTHONHASHSEED=0;
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index ea47d41fc348..ae7e30d0ff89 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -6162,6 +6162,9 @@ with pkgs;
     self = python36;
   };
 
+  # Should eventually be moved inside Python interpreters.
+  python-setup-hook = callPackage ../development/interpreters/python/setup-hook.nix { };
+
   pypy = pypy27;
 
   pypy27 = callPackage ../development/interpreters/python/pypy/2.7 {