From 7fd749009f63569a6e862527dcea661495277e90 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Apr 2022 17:00:36 -0700 Subject: [PATCH 1/5] stdenv: force gmp to rebuild in stage4 of the bootstrap As explained in the comment, this ensures that stage4-coreutils does not leak a reference to the bootstrap-files by way of libgmp. This will allow the next patch in this series to build stage4-coreutils using a dynamically-linked (rather than statically-linked) libgmp. --- pkgs/stdenv/linux/default.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index d625ab5b3013..e10393dd655c 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -336,7 +336,7 @@ in # because gcc (since JAR support) already depends on zlib, and # then if we already have a zlib we want to use that for the # other purposes (binutils and top-level pkgs) too. - inherit (prevStage) gettext gnum4 bison gmp perl texinfo zlib linuxHeaders libidn2 libunistring; + inherit (prevStage) gettext gnum4 bison perl texinfo zlib linuxHeaders libidn2 libunistring; ${localSystem.libc} = getLibc prevStage; binutils = super.binutils.override { # Don't use stdenv's shell but our own @@ -347,6 +347,11 @@ in }; }; + # force gmp to rebuild so we have the option of dynamically linking + # libgmp without creating a reference path from: + # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap + gmp = super.gmp.override { stdenv = self.stdenv; }; + gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; @@ -417,7 +422,7 @@ in # Simple executable tools concatMap (p: [ (getBin p) (getLib p) ]) [ gzip bzip2 xz bash binutils.bintools coreutils diffutils findutils - gawk gnumake gnused gnutar gnugrep gnupatch patchelf ed file + gawk gmp gnumake gnused gnutar gnugrep gnupatch patchelf ed file ] # Library dependencies ++ map getLib ( From 122b6930b0b5b8aa9911c6025ed12a7e5a8b9fab Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Apr 2022 16:07:01 -0700 Subject: [PATCH 2/5] stdenv: cause makeStaticLibraries usage to agree with usage spec The usage of `makeStaticLibraries` in stdenv/linux/default.nix is prefaced by this comment: # Link GCC statically against GMP etc. This makes sense because # these builds of the libraries are only used by GCC, so it # reduces the size of the stdenv closure. However "these builds of the libraries are only used by GCC" is not actually true. As currently written, the stage4 coreutils links against these customized, static-ified libraries. Beside the fact that the code doesn't actually do what it says, this causes other problems as well. One example is #168983, which arises because have a dynamically-linked binary (coreutils) which is built from statically-linked libraries (libgmp.a); doing this causes mayhem on platforms where `-fstack-protector` needs an auxiliary `libssp.{so,a}` library; we end up with link failures because some parts of the resulting binary want `libssp.so` and other parts want `libssp_nonshared.a`. Let's make the code actually do what the comment says, by moving these definitions into the `gcc-unwrapped` override. This will cause the stage4-coreutils to link against libgmp dynamically, rather than statically. For this reason this commit depends on the previous commit, which allows that to be done without creating a forbidden reference from stdenv-final to the bootstrap-files. --- pkgs/stdenv/linux/default.nix | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index e10393dd655c..490840f254cb 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -304,15 +304,14 @@ in binutils coreutils gnugrep perl patchelf linuxHeaders gnum4 bison libidn2 libunistring; ${localSystem.libc} = getLibc prevStage; - # Link GCC statically against GMP etc. This makes sense because - # these builds of the libraries are only used by GCC, so it - # reduces the size of the stdenv closure. - gmp = super.gmp.override { stdenv = self.makeStaticLibraries self.stdenv; }; - mpfr = super.mpfr.override { stdenv = self.makeStaticLibraries self.stdenv; }; - libmpc = super.libmpc.override { stdenv = self.makeStaticLibraries self.stdenv; }; - isl_0_20 = super.isl_0_20.override { stdenv = self.makeStaticLibraries self.stdenv; }; gcc-unwrapped = super.gcc-unwrapped.override { - isl = isl_0_20; + # Link GCC statically against GMP etc. This makes sense because + # these builds of the libraries are only used by GCC, so it + # reduces the size of the stdenv closure. + gmp = super.gmp.override { stdenv = self.makeStaticLibraries self.stdenv; }; + mpfr = super.mpfr.override { stdenv = self.makeStaticLibraries self.stdenv; }; + libmpc = super.libmpc.override { stdenv = self.makeStaticLibraries self.stdenv; }; + isl = super.isl_0_20.override { stdenv = self.makeStaticLibraries self.stdenv; }; # Use a deterministically built compiler # see https://github.com/NixOS/nixpkgs/issues/108475 for context reproducibleBuild = true; From 23ea8b35dacd9152c9e0e21577d5afe3e39b6255 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Apr 2022 17:04:58 -0700 Subject: [PATCH 3/5] stdenv: label the ephemeral coreutils-stage4 package During stdenv bootstrapping, coreutils is built twice. This makes troubleshooting very difficult, because both packages have name="coreutils", so it is a hassle to figure out "which coreutils am I using / is not building"? The first of these builds is used only in stage4, and is not part of the final stdenv. Let's label that one with a different `name` attribute to make it obvious which is which. --- pkgs/stdenv/linux/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 490840f254cb..ad5a91f9340e 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -351,6 +351,9 @@ in # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap gmp = super.gmp.override { stdenv = self.stdenv; }; + # coreutils gets rebuilt both here and also in the final stage; we rename this one to avoid confusion + coreutils = super.coreutils.overrideAttrs (a: a // { name = "coreutils-stage4"; }); + gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; From a9e0d864119c0313ece2cbb836c71821818e38c1 Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+a-m-joseph@users.noreply.github.com> Date: Wed, 20 Apr 2022 22:45:14 +0000 Subject: [PATCH 4/5] Update pkgs/stdenv/linux/default.nix Co-authored-by: sternenseemann --- pkgs/stdenv/linux/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index ad5a91f9340e..4983c819216a 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -352,7 +352,7 @@ in gmp = super.gmp.override { stdenv = self.stdenv; }; # coreutils gets rebuilt both here and also in the final stage; we rename this one to avoid confusion - coreutils = super.coreutils.overrideAttrs (a: a // { name = "coreutils-stage4"; }); + coreutils = super.coreutils.overrideAttrs (_: { pname = "coreutils-stage4"; }); gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; From 02630180fad510ee877fa51112b7c7b230ef2f13 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Wed, 27 Apr 2022 00:27:19 -0700 Subject: [PATCH 5/5] stdenv: add -stageX markers to gmp, mpfr, libmpc, and isl --- pkgs/stdenv/linux/default.nix | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 4983c819216a..4e68b8c9a50b 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -304,14 +304,18 @@ in binutils coreutils gnugrep perl patchelf linuxHeaders gnum4 bison libidn2 libunistring; ${localSystem.libc} = getLibc prevStage; - gcc-unwrapped = super.gcc-unwrapped.override { + gcc-unwrapped = + let makeStaticLibrariesAndMark = pkg: + lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; }) + .overrideAttrs (a: { pname = "${a.pname}-stage3"; }); + in super.gcc-unwrapped.override { # Link GCC statically against GMP etc. This makes sense because # these builds of the libraries are only used by GCC, so it # reduces the size of the stdenv closure. - gmp = super.gmp.override { stdenv = self.makeStaticLibraries self.stdenv; }; - mpfr = super.mpfr.override { stdenv = self.makeStaticLibraries self.stdenv; }; - libmpc = super.libmpc.override { stdenv = self.makeStaticLibraries self.stdenv; }; - isl = super.isl_0_20.override { stdenv = self.makeStaticLibraries self.stdenv; }; + gmp = makeStaticLibrariesAndMark super.gmp; + mpfr = makeStaticLibrariesAndMark super.mpfr; + libmpc = makeStaticLibrariesAndMark super.libmpc; + isl = makeStaticLibrariesAndMark super.isl_0_20; # Use a deterministically built compiler # see https://github.com/NixOS/nixpkgs/issues/108475 for context reproducibleBuild = true; @@ -349,10 +353,10 @@ in # force gmp to rebuild so we have the option of dynamically linking # libgmp without creating a reference path from: # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap - gmp = super.gmp.override { stdenv = self.stdenv; }; + gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; }); # coreutils gets rebuilt both here and also in the final stage; we rename this one to avoid confusion - coreutils = super.coreutils.overrideAttrs (_: { pname = "coreutils-stage4"; }); + coreutils = super.coreutils.overrideAttrs (a: { pname = "${a.pname}-stage4"; }); gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false;