CPython: merge expressions of interpreters

Each time a new major/minor version of CPython was released, a new
expression would be written, typically copied from the previous release.
Often fixes are only made in the current/latest release. By merging the
expressions it's more likely that modifications end up in all versions,
as is likely intended.

This commit introduces one expression for Python 3, and another for 2.7.
These two may also be merged, but it will result in a lot of extra
conditionals making the expression harder to follow.

A common passthru is introduced for CPython and PyPy.

python 2.7: use common passthru

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -1,7 +1,9 @@
-{ stdenv, buildPackages, fetchurl
+{ stdenv, fetchurl, fetchpatch
 , bzip2
+, expat
+, libffi
 , gdbm
-, fetchpatch
+, db
 , ncurses
 , openssl
 , readline
@@ -10,15 +12,16 @@
 , zlib
 , callPackage
 , self
-, db
-, expat
-, libffi
 , CF, configd, coreutils
 , python-setup-hook
 # Some proprietary libs assume UCS2 unicode, especially on darwin :(
 , ucsEncoding ? 4
 # For the Python package set
 , packageOverrides ? (self: super: {})
+, buildPackages
+, sourceVersion
+, sha256
+, passthruFun
 }:
 
 assert x11Support -> tcl != null
@@ -29,16 +32,23 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
-  majorVersion = "2.7";
+  passthru = passthruFun rec {
-  minorVersion = "15";
+    inherit self sourceVersion packageOverrides;
-  minorVersionSuffix = "";
+    implementation = "cpython";
-  version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
+    libPrefix = "python${pythonVersion}";
-  libPrefix = "python${majorVersion}";
+    executable = libPrefix;
-  sitePackages = "lib/${libPrefix}/site-packages";
+    pythonVersion = with sourceVersion; "${major}.${minor}";
+    sitePackages = "lib/${libPrefix}/site-packages";
+    inherit pythonForBuild;
+  } // {
+    inherit ucsEncoding;
+  };
+
+  version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
 
   src = fetchurl {
-    url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
+    url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
-    sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92";
+    inherit sha256;
   };
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
@@ -191,12 +201,11 @@ let
   # Build the basic Python interpreter without modules that have
   # external dependencies.
 
-in stdenv.mkDerivation ({
+in with passthru; stdenv.mkDerivation ({
-    name = "python-${version}";
+    pname = "python";
-    pythonVersion = majorVersion;
+    inherit version;
 
-    inherit majorVersion version src patches buildInputs nativeBuildInputs
+    inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags;
-            preConfigure configureFlags;
 
     LDFLAGS = stdenv.lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
     inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
@@ -215,7 +224,7 @@ in stdenv.mkDerivation ({
       ''
         # needed for some packages, especially packages that backport
         # functionality to 2.x from 3.x
-        for item in $out/lib/python${majorVersion}/test/*; do
+        for item in $out/lib/${libPrefix}/test/*; do
           if [[ "$item" != */test_support.py*
              && "$item" != */test/support
              && "$item" != */test/regrtest.py* ]]; then
@@ -224,9 +233,9 @@ in stdenv.mkDerivation ({
             echo $item
           fi
         done
-        touch $out/lib/python${majorVersion}/test/__init__.py
+        touch $out/lib/${libPrefix}/test/__init__.py
-        ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb
+        ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb
-        ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion}
+        ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor}
         ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
 
         # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
@@ -249,21 +258,7 @@ in stdenv.mkDerivation ({
         cp libpython2.7.dll.a $out/lib
       '';
 
-    passthru = let
+    inherit passthru;
-      pythonPackages = callPackage ../../../../../top-level/python-packages.nix {
-        python = self;
-        overrides = packageOverrides;
-      };
-    in rec {
-      inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch ucsEncoding;
-      executable = libPrefix;
-      buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-      withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-      pkgs = pythonPackages;
-      isPy2 = true;
-      isPy27 = true;
-      interpreter = "${self}/bin/${executable}";
-    };
 
     enableParallelBuilding = true;
 

pkgs/development/interpreters/python/cpython/3.5/default.nix

@@ -1,212 +0,0 @@
-{ stdenv, fetchurl, fetchpatch
-, bzip2
-, expat
-, libffi
-, gdbm
-, lzma
-, ncurses
-, openssl
-, readline
-, sqlite
-, tcl ? null, tk ? null, tix ? null, libX11 ? null, xproto ? null, x11Support ? false
-, zlib
-, callPackage
-, self
-, CF, configd
-, python-setup-hook
-# For the Python package set
-, packageOverrides ? (self: super: {})
-}:
-
-assert x11Support -> tcl != null
-                  && tk != null
-                  && xproto != null
-                  && libX11 != null;
-
-with stdenv.lib;
-
-let
-  majorVersion = "3.5";
-  minorVersion = "6";
-  minorVersionSuffix = "";
-  version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
-  libPrefix = "python${majorVersion}";
-  sitePackages = "lib/${libPrefix}/site-packages";
-
-  buildInputs = filter (p: p != null) [
-    zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
-    ++ optionals x11Support [ tcl tk libX11 xproto ]
-    ++ optionals stdenv.isDarwin [ CF configd ];
-
-  hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
-
-in stdenv.mkDerivation {
-  name = "python3-${version}";
-  pythonVersion = majorVersion;
-  inherit majorVersion version;
-
-  inherit buildInputs;
-
-  src = fetchurl {
-    url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
-    sha256 = "0pqmf51zy2lzhbaj4yya2py2qr653j9152d0rg3p7wi1yl2dwp7m";
-  };
-
-  NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
-
-  # Determinism: The interpreter is patched to write null timestamps when compiling python files.
-  # This way python doesn't try to update them when we freeze timestamps in nix store.
-  DETERMINISTIC_BUILD=1;
-  # Determinism: We fix the hashes of str, bytes and datetime objects.
-  PYTHONHASHSEED=0;
-
-  prePatch = optionalString stdenv.isDarwin ''
-    substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
-    substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
-  '';
-
-  patches = [
-    ./no-ldconfig.patch
-    ./ld_library_path.patch
-  ] ++ optionals stdenv.isDarwin [
-    # Fix for https://bugs.python.org/issue24658
-    (fetchpatch {
-      url = "https://bugs.python.org/file45178/issue24658-3-3.6.diff";
-      sha256 = "1x060hs80nl34mcl2ji2i7l4shxkmxwgq8h8lcmav8rjqqz1nb4a";
-    })
-  ] ++ optionals (x11Support && stdenv.isDarwin) [
-    ./use-correct-tcl-tk-on-darwin.patch
-  ] ++ optionals hasDistutilsCxxPatch [
-    # Fix for http://bugs.python.org/issue1222585
-    # Upstream distutils is calling C compiler to compile C++ code, which
-    # only works for GCC and Apple Clang. This makes distutils to call C++
-    # compiler when needed.
-    (fetchpatch {
-      url = "https://bugs.python.org/file47046/python-3.x-distutils-C++.patch";
-      sha256 = "0dgdn9k2kmw4wh90vdnjcrnn97ylxgx7mbn9l87fwz6j501jqvk8";
-      extraPrefix = "";
-    })
-  ];
-
-  postPatch = ''
-    # Determinism
-    substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
-    # Determinism. This is done unconditionally
-    substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
-  '' + optionalString (x11Support && (tix != null)) ''
-    substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
-  '';
-
-  CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
-  LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
-  LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
-
-  configureFlags = [
-    "--enable-shared"
-    "--with-threads"
-    "--without-ensurepip"
-    "--with-system-expat"
-    "--with-system-ffi"
-  ]
-    # Never even try to use lchmod on linux,
-    # don't rely on detecting glibc-isms.
-  ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no";
-
-  preConfigure = ''
-    for i in /usr /sw /opt /pkg; do	# improve purity
-      substituteInPlace ./setup.py --replace $i /no-such-path
-    done
-    ${optionalString stdenv.isDarwin ''
-       export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
-       export MACOSX_DEPLOYMENT_TARGET=10.6
-     ''
-     + optionalString stdenv.hostPlatform.isMusl ''
-      export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
-     ''}
-  '';
-
-  setupHook = python-setup-hook sitePackages;
-
-  postInstall = ''
-    # needed for some packages, especially packages that backport functionality
-    # to 2.x from 3.x
-    for item in $out/lib/python${majorVersion}/test/*; do
-      if [[ "$item" != */test_support.py*
-         && "$item" != */test/support
-         && "$item" != */test/libregrtest
-         && "$item" != */test/regrtest.py* ]]; then
-        rm -rf "$item"
-      else
-        echo $item
-      fi
-    done
-    touch $out/lib/python${majorVersion}/test/__init__.py
-
-    ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
-
-    # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
-    echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
-
-    # Determinism: Windows installers were not deterministic.
-    # We're also not interested in building Windows installers.
-    find "$out" -name 'wininst*.exe' | xargs -r rm -f
-
-    # Use Python3 as default python
-    ln -s "$out/bin/idle3" "$out/bin/idle"
-    ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
-    ln -s "$out/bin/python3" "$out/bin/python"
-    ln -s "$out/bin/python3-config" "$out/bin/python-config"
-    ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
-
-    # Get rid of retained dependencies on -dev packages, and remove
-    # some $TMPDIR references to improve binary reproducibility.
-    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
-    for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
-      sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
-    done
-
-    # Determinism: rebuild all bytecode
-    # We exclude lib2to3 because that's Python 2 code which fails
-    # We rebuild three times, once for each optimization level
-    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
-  '';
-
-  passthru = let
-    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {
-      python = self;
-      overrides = packageOverrides;
-    };
-  in rec {
-    inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch;
-    executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-    pkgs = pythonPackages;
-    isPy3 = true;
-    isPy35 = true;
-    interpreter = "${self}/bin/${executable}";
-  };
-
-  enableParallelBuilding = true;
-
-  doCheck = false; # expensive, and fails
-
-  meta = {
-    homepage = http://python.org;
-    description = "A high-level dynamically-typed programming language";
-    longDescription = ''
-      Python is a remarkably powerful dynamic programming language that
-      is used in a wide variety of application domains. Some of its key
-      distinguishing features include: clear, readable syntax; strong
-      introspection capabilities; intuitive object orientation; natural
-      expression of procedural code; full modularity, supporting
-      hierarchical packages; exception-based error handling; and very
-      high level dynamic data types.
-    '';
-    license = licenses.psfl;
-    platforms = with platforms; linux ++ darwin;
-    maintainers = with maintainers; [ fridh ];
-  };
-}

pkgs/development/interpreters/python/cpython/3.6/default.nix

@@ -1,228 +0,0 @@
-{ stdenv, fetchurl, fetchpatch, buildPackages
-, bzip2
-, expat
-, libffi
-, gdbm
-, lzma
-, ncurses
-, openssl
-, readline
-, sqlite
-, tcl ? null, tk ? null, tix ? null, libX11 ? null, xproto ? null, x11Support ? false
-, zlib
-, callPackage
-, self
-, CF, configd
-, python-setup-hook
-# For the Python package set
-, packageOverrides ? (self: super: {})
-}:
-
-assert x11Support -> tcl != null
-                  && tk != null
-                  && xproto != null
-                  && libX11 != null;
-with stdenv.lib;
-
-let
-  majorVersion = "3.6";
-  minorVersion = "8";
-  minorVersionSuffix = "";
-  version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
-  libPrefix = "python${majorVersion}";
-  sitePackages = "lib/${libPrefix}/site-packages";
-
-  buildInputs = filter (p: p != null) [
-    zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
-    ++ optionals x11Support [ tcl tk libX11 xproto ]
-    ++ optionals stdenv.isDarwin [ CF configd ];
-
-  nativeBuildInputs =
-    optional (stdenv.hostPlatform != stdenv.buildPlatform) buildPackages.python3;
-
-  hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
-
-in stdenv.mkDerivation {
-  name = "python3-${version}";
-  pythonVersion = majorVersion;
-  inherit majorVersion version;
-
-  inherit buildInputs nativeBuildInputs;
-
-  src = fetchurl {
-    url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
-    sha256 = "14qi6n5gpcjnwy165wi9hkfcmbadc95ny6bxxldknxwmx50n4i1m";
-  };
-
-  NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
-
-  # Determinism: The interpreter is patched to write null timestamps when compiling python files.
-  # This way python doesn't try to update them when we freeze timestamps in nix store.
-  DETERMINISTIC_BUILD=1;
-  # Determinism: We fix the hashes of str, bytes and datetime objects.
-  PYTHONHASHSEED=0;
-
-  prePatch = optionalString stdenv.isDarwin ''
-    substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
-    substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
-  '';
-
-  patches = [
-    ./no-ldconfig.patch
-  ] ++ optionals (x11Support && stdenv.isDarwin) [
-    ./use-correct-tcl-tk-on-darwin.patch
-  ] ++ optionals hasDistutilsCxxPatch [
-    # Fix for http://bugs.python.org/issue1222585
-    # Upstream distutils is calling C compiler to compile C++ code, which
-    # only works for GCC and Apple Clang. This makes distutils to call C++
-    # compiler when needed.
-    (fetchpatch {
-      url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch";
-      sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2";
-    })
-  ];
-
-  postPatch = ''
-    # Determinism
-    substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
-    # Determinism. This is done unconditionally
-    substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
-  '' + optionalString (x11Support && (tix != null)) ''
-    substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
-  '';
-
-  CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
-  LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
-  LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
-
-  configureFlags = [
-    "--enable-shared"
-    "--with-threads"
-    "--without-ensurepip"
-    "--with-system-expat"
-    "--with-system-ffi"
-  ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-    "ac_cv_buggy_getaddrinfo=no"
-    # Assume little-endian IEEE 754 floating point when cross compiling
-    "ac_cv_little_endian_double=yes"
-    "ac_cv_big_endian_double=no"
-    "ac_cv_mixed_endian_double=no"
-    "ac_cv_x87_double_rounding=yes"
-    "ac_cv_tanh_preserves_zero_sign=yes"
-    # Generally assume that things are present and work
-    "ac_cv_posix_semaphores_enabled=yes"
-    "ac_cv_broken_sem_getvalue=no"
-    "ac_cv_wchar_t_signed=yes"
-    "ac_cv_rshift_extends_sign=yes"
-    "ac_cv_broken_nice=no"
-    "ac_cv_broken_poll=no"
-    "ac_cv_working_tzset=yes"
-    "ac_cv_have_long_long_format=yes"
-    "ac_cv_have_size_t_format=yes"
-    "ac_cv_computed_gotos=yes"
-    "ac_cv_file__dev_ptmx=yes"
-    "ac_cv_file__dev_ptc=yes"
-  ]
-    # Never even try to use lchmod on linux,
-    # don't rely on detecting glibc-isms.
-  ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no";
-
-  preConfigure = ''
-    for i in /usr /sw /opt /pkg; do	# improve purity
-      substituteInPlace ./setup.py --replace $i /no-such-path
-    done
-    ${optionalString stdenv.isDarwin ''
-       export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
-       export MACOSX_DEPLOYMENT_TARGET=10.6
-     ''
-     + optionalString stdenv.hostPlatform.isMusl ''
-      export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
-     ''}
-  '';
-
-  setupHook = python-setup-hook sitePackages;
-
-  postInstall = ''
-    # needed for some packages, especially packages that backport functionality
-    # to 2.x from 3.x
-    for item in $out/lib/python${majorVersion}/test/*; do
-      if [[ "$item" != */test_support.py*
-         && "$item" != */test/support
-         && "$item" != */test/libregrtest
-         && "$item" != */test/regrtest.py* ]]; then
-        rm -rf "$item"
-      else
-        echo $item
-      fi
-    done
-    touch $out/lib/python${majorVersion}/test/__init__.py
-
-    ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
-
-    # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
-    echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
-
-    # Determinism: Windows installers were not deterministic.
-    # We're also not interested in building Windows installers.
-    find "$out" -name 'wininst*.exe' | xargs -r rm -f
-
-    # Use Python3 as default python
-    ln -s "$out/bin/idle3" "$out/bin/idle"
-    ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
-    ln -s "$out/bin/python3" "$out/bin/python"
-    ln -s "$out/bin/python3-config" "$out/bin/python-config"
-    ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
-
-    # Get rid of retained dependencies on -dev packages, and remove
-    # some $TMPDIR references to improve binary reproducibility.
-    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
-    for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do
-      sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
-    done
-  '' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform) ''
-    # Determinism: rebuild all bytecode
-    # We exclude lib2to3 because that's Python 2 code which fails
-    # We rebuild three times, once for each optimization level
-    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
-  '';
-
-  passthru = let
-    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {
-      python = self;
-      overrides = packageOverrides;
-    };
-  in rec {
-    inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch;
-    executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-    pkgs = pythonPackages;
-    isPy3 = true;
-    isPy36 = true;
-    is_py3k = true;  # deprecated
-    interpreter = "${self}/bin/${executable}";
-  };
-
-  enableParallelBuilding = true;
-
-  doCheck = false; # expensive, and fails
-
-  meta = {
-    homepage = http://python.org;
-    description = "A high-level dynamically-typed programming language";
-    longDescription = ''
-      Python is a remarkably powerful dynamic programming language that
-      is used in a wide variety of application domains. Some of its key
-      distinguishing features include: clear, readable syntax; strong
-      introspection capabilities; intuitive object orientation; natural
-      expression of procedural code; full modularity, supporting
-      hierarchical packages; exception-based error handling; and very
-      high level dynamic data types.
-    '';
-    license = licenses.psfl;
-    platforms = with platforms; linux ++ darwin;
-    maintainers = with maintainers; [ fridh kragniz ];
-  };
-}

pkgs/development/interpreters/python/cpython/default.nix

@@ -18,6 +18,9 @@
 # For the Python package set
 , packageOverrides ? (self: super: {})
 , buildPackages
+, sourceVersion
+, sha256
+, passthruFun
 }:
 
 assert x11Support -> tcl != null
@@ -27,12 +30,24 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
-  majorVersion = "3.7";
+
-  minorVersion = "2";
+  passthru = passthruFun rec {
-  minorVersionSuffix = "";
+    inherit self sourceVersion packageOverrides;
-  version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
+    implementation = "cpython";
-  libPrefix = "python${majorVersion}";
+    libPrefix = "python${pythonVersion}";
-  sitePackages = "lib/${libPrefix}/site-packages";
+    executable = libPrefix;
+    pythonVersion = with sourceVersion; "${major}.${minor}";
+    sitePackages = "lib/${libPrefix}/site-packages";
+    inherit pythonForBuild;
+  };
+
+  version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
+
+  nativeBuildInputs = [
+    nukeReferences
+  ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+    buildPackages.stdenv.cc crossPython
+  ];
 
   buildInputs = filter (p: p != null) [
     zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
@@ -40,44 +55,46 @@ let
     ++ optionals stdenv.isDarwin [ CF configd ];
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
+
+  crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+
   pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then
     "$out/bin/python"
-  else
+  else crossPython.interpreter;
-    buildPackages.python37.interpreter;
-in stdenv.mkDerivation {
-  name = "python3-${version}";
-  pythonVersion = majorVersion;
-  inherit majorVersion version;
 
-  inherit buildInputs;
+in with passthru; stdenv.mkDerivation {
+  pname = "python3";
+  inherit version;
 
-  nativeBuildInputs = [ nukeReferences ] ++
+  inherit buildInputs nativeBuildInputs;
-    optionals (stdenv.hostPlatform != stdenv.buildPlatform)
-    [ buildPackages.stdenv.cc buildPackages.python37 ];
 
   src = fetchurl {
-    url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
+    url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
-    sha256 = "1fzi9d2gibh0wzwidyckzbywsxcsbckgsl05ryxlifxia77fhgyq";
+    inherit sha256;
   };
 
-  NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
-
-  # Determinism: We fix the hashes of str, bytes and datetime objects.
-  PYTHONHASHSEED=0;
-
   prePatch = optionalString stdenv.isDarwin ''
     substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
     substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
   '';
 
   patches = [
-    ./no-ldconfig.patch
+    # Disable the use of ldconfig in ctypes.util.find_library (since
+    # ldconfig doesn't work on NixOS), and don't use
+    # ctypes.util.find_library during the loading of the uuid module
+    # (since it will do a futile invocation of gcc (!) to find
+    # libuuid, slowing down program startup a lot).
+    (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch")
+  ] ++ optionals isPy35 [
+    # Backports support for LD_LIBRARY_PATH from 3.6
+    ./3.5/ld_library_path.patch
+  ] ++ optionals isPy37 [
     # Fix darwin build https://bugs.python.org/issue34027
     (fetchpatch {
       url = https://bugs.python.org/file47666/darwin-libutil.patch;
       sha256 = "0242gihnw3wfskl4fydp2xanpl8k5q7fj4dp7dbbqf46a4iwdzpa";
     })
-  ] ++ optionals hasDistutilsCxxPatch [
+  ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [
     # Fix for http://bugs.python.org/issue1222585
     # Upstream distutils is calling C compiler to compile C++ code, which
     # only works for GCC and Apple Clang. This makes distutils to call C++
@@ -93,9 +110,12 @@ in stdenv.mkDerivation {
     substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
   '';
 
-  CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
+  CPPFLAGS = "${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
-  LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
+  LDFLAGS = "${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
-  LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
+  LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
+  NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
+  # Determinism: We fix the hashes of str, bytes and datetime objects.
+  PYTHONHASHSEED=0;
 
   configureFlags = [
     "--enable-shared"
@@ -125,16 +145,25 @@ in stdenv.mkDerivation {
     "ac_cv_computed_gotos=yes"
     "ac_cv_file__dev_ptmx=yes"
     "ac_cv_file__dev_ptc=yes"
+  ] ++ optionals stdenv.hostPlatform.isLinux [
+    # Never even try to use lchmod on linux,
+    # don't rely on detecting glibc-isms.
+    "ac_cv_func_lchmod=no"
   ];
 
   preConfigure = ''
     for i in /usr /sw /opt /pkg; do	# improve purity
       substituteInPlace ./setup.py --replace $i /no-such-path
     done
-    ${optionalString stdenv.isDarwin ''
+  '' + optionalString stdenv.isDarwin ''
-       export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
+    export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
-       export MACOSX_DEPLOYMENT_TARGET=10.6
+    export MACOSX_DEPLOYMENT_TARGET=10.6
-     ''}
+  '' + optionalString (isPy3k && pythonOlder "3.7") ''
+    # Determinism: The interpreter is patched to write null timestamps when compiling python files.
+    # This way python does not try to update them when we freeze timestamps in nix store.
+    export DETERMINISTIC_BUILD=1;
+  '' + optionalString stdenv.hostPlatform.isMusl ''
+    export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
   '';
 
   setupHook = python-setup-hook sitePackages;
@@ -142,7 +171,7 @@ in stdenv.mkDerivation {
   postInstall = ''
     # needed for some packages, especially packages that backport functionality
     # to 2.x from 3.x
-    for item in $out/lib/python${majorVersion}/test/*; do
+    for item in $out/lib/${libPrefix}/test/*; do
       if [[ "$item" != */test_support.py*
          && "$item" != */test/support
          && "$item" != */test/libregrtest
@@ -152,9 +181,9 @@ in stdenv.mkDerivation {
         echo $item
       fi
     done
-    touch $out/lib/python${majorVersion}/test/__init__.py
+    touch $out/lib/${libPrefix}/test/__init__.py
 
-    ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
+    ln -s "$out/include/${executable}m" "$out/include/${executable}"
 
     # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
     echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
@@ -173,14 +202,13 @@ in stdenv.mkDerivation {
     # Get rid of retained dependencies on -dev packages, and remove
     # some $TMPDIR references to improve binary reproducibility.
     # Note that the .pyc file of _sysconfigdata.py should be regenerated!
-    for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do
+    for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do
-      sed -i $i -e "s|$TMPDIR|/no-such-path|g"
+       sed -i $i -e "s|$TMPDIR|/no-such-path|g"
-      nuke-refs $i
     done
 
     # Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668
-    find $out/lib/python*/config-*-* -type f -print -exec nuke-refs '{}' +
+    find $out/lib/python*/config-* -type f -print -exec nuke-refs '{}' +
-    find $out/lib -name '_sysconfigdata_m*.py*' -print -exec nuke-refs '{}' +
+    find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs '{}' +
 
     # Determinism: rebuild all bytecode
     # We exclude lib2to3 because that's Python 2 code which fails
@@ -196,22 +224,7 @@ in stdenv.mkDerivation {
   # explicitly specify in our configure flags above.
   disallowedReferences = [ openssl.dev ];
 
-  passthru = let
+  inherit passthru;
-    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {
-      python = self;
-      overrides = packageOverrides;
-    };
-  in rec {
-    inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch;
-    executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-    pkgs = pythonPackages;
-    isPy3 = true;
-    isPy37 = true;
-    is_py3k = true;  # deprecated
-    interpreter = "${self}/bin/${executable}";
-  };
 
   enableParallelBuilding = true;
 
@@ -229,6 +242,6 @@ in stdenv.mkDerivation {
     '';
     license = licenses.psfl;
     platforms = with platforms; linux ++ darwin;
-    maintainers = with maintainers; [ fridh kragniz ];
+    maintainers = with maintainers; [ fridh ];
   };
 }

pkgs/development/interpreters/python/default.nix

@@ -0,0 +1,106 @@
+{ pkgs, lib }:
+
+with pkgs;
+
+(let
+
+  # Common passthru for all Python interpreters.
+  passthruFun =
+    { implementation
+    , libPrefix
+    , executable
+    , sourceVersion
+    , pythonVersion
+    , packageOverrides
+    , sitePackages
+    , pythonForBuild
+    , self
+    }: let
+      pythonPackages = callPackage ../../../top-level/python-packages.nix {
+        python = self;
+        overrides = packageOverrides;
+      };
+    in rec {
+        sitePackages = "lib/${libPrefix}/site-packages";
+        isPy27 = pythonVersion == "2.7";
+        isPy33 = pythonVersion == "3.3"; # TODO: remove
+        isPy34 = pythonVersion == "3.4"; # TODO: remove
+        isPy35 = pythonVersion == "3.5";
+        isPy36 = pythonVersion == "3.6";
+        isPy37 = pythonVersion == "3.7";
+        isPy3k = lib.strings.substring 0 1 pythonVersion == "3";
+        isPyPy = interpreter == "pypy";
+
+        buildEnv = callPackage ./wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
+        withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;};
+        pkgs = pythonPackages;
+        interpreter = "${self}/bin/${executable}";
+        inherit executable implementation libPrefix pythonVersion;
+        inherit sourceVersion;
+        pythonAtLeast = lib.versionAtLeast pythonVersion;
+        pythonOlder = lib.versionOlder pythonVersion;
+        inherit pythonForBuild;
+
+        majorVersion = pythonVersion; # Backwards compatibility. Should be dropped.
+  };
+
+in {
+
+  python27 = callPackage ./cpython/2.7 {
+    self = python27;
+    sourceVersion = {
+      major = "2";
+      minor = "7";
+      patch = "15";
+      suffix = "";
+    };
+    sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92";
+    inherit (darwin) CF configd;
+    inherit passthruFun;
+  };
+
+  python35 = callPackage ./cpython {
+    self = python35;
+    sourceVersion = {
+      major = "3";
+      minor = "5";
+      patch = "6";
+      suffix = "";
+    };
+    sha256 = "0pqmf51zy2lzhbaj4yya2py2qr653j9152d0rg3p7wi1yl2dwp7m";
+    inherit (darwin) CF configd;
+    inherit passthruFun;
+  };
+
+  python36 = callPackage ./cpython {
+    self = python36;
+    sourceVersion = {
+      major = "3";
+      minor = "6";
+      patch = "8";
+      suffix = "";
+    };
+    sha256 = "14qi6n5gpcjnwy165wi9hkfcmbadc95ny6bxxldknxwmx50n4i1m";
+    inherit (darwin) CF configd;
+    inherit passthruFun;
+  };
+
+  python37 = callPackage ./cpython {
+    self = python37;
+    sourceVersion = {
+      major = "3";
+      minor = "7";
+      patch = "2";
+      suffix = "";
+    };
+    sha256 = "1fzi9d2gibh0wzwidyckzbywsxcsbckgsl05ryxlifxia77fhgyq";
+    inherit (darwin) CF configd;
+    inherit passthruFun;
+  };
+
+#   pypy27 = callPackage ./pypy {
+#     self = pypy27;
+#     implementation = "pypy";
+#   };
+
+})

pkgs/top-level/all-packages.nix

@@ -7953,22 +7953,8 @@ in
   python2Packages = python2.pkgs;
   python3Packages = python3.pkgs;
 
-  python27 = callPackage ../development/interpreters/python/cpython/2.7 {
+  pythonInterpreters = callPackage ./../development/interpreters/python {};
-    self = python27;
+  inherit (pythonInterpreters) python27 python35 python36 python37;
-    inherit (darwin) CF configd;
-  };
-  python35 = callPackage ../development/interpreters/python/cpython/3.5 {
-    inherit (darwin) CF configd;
-    self = python35;
-  };
-  python36 = callPackage ../development/interpreters/python/cpython/3.6 {
-    inherit (darwin) CF configd;
-    self = python36;
-  };
-  python37 = callPackage ../development/interpreters/python/cpython/3.7 {
-    inherit (darwin) CF configd;
-    self = python37;
-  };
 
   pypy27 = callPackage ../development/interpreters/python/pypy/2.7 {
     self = pypy27;

pkgs/top-level/python-packages.nix

@@ -18,16 +18,7 @@ let
   packages = ( self:
 
 let
-  pythonAtLeast = versionAtLeast python.pythonVersion;
+  inherit (python.passthru) isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPy3k isPyPy pythonAtLeast pythonOlder;
-  pythonOlder = versionOlder python.pythonVersion;
-  isPy27 = python.pythonVersion == "2.7";
-  isPy33 = python.pythonVersion == "3.3";
-  isPy34 = python.pythonVersion == "3.4";
-  isPy35 = python.pythonVersion == "3.5";
-  isPy36 = python.pythonVersion == "3.6";
-  isPy37 = python.pythonVersion == "3.7";
-  isPyPy = strings.substring 0 4 python.executable == "pypy";
-  isPy3k = strings.substring 0 1 python.pythonVersion == "3";
 
   callPackage = pkgs.newScope self;
 
@@ -130,7 +121,8 @@ let
 
 in {
 
-  inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPyPy isPy3k buildPythonPackage buildPythonApplication;
+  inherit (python.passthru) isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPy3k isPyPy pythonAtLeast pythonOlder;
+  inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
   inherit fetchPypi callPackage;
   inherit hasPythonModule requiredPythonModules makePythonPath disabledIf;
   inherit toPythonModule toPythonApplication;