From 005cfc8b0cea9cef36a8d129c3b06e0a98a918da Mon Sep 17 00:00:00 2001 From: Emily Trau Date: Thu, 18 May 2023 10:12:53 +1000 Subject: [PATCH 1/3] minimal-bootstrap: create top-level attr for bootstrap sources --- .../stage0-posix/bootstrap-sources.nix | 22 ++++++++++++--- .../minimal-bootstrap/stage0-posix/hex0.nix | 2 +- .../stage0-posix/kaem/minimal.nix | 2 +- .../stage0-posix/make-bootstrap-sources.nix | 28 ++++++++++++++----- pkgs/top-level/all-packages.nix | 1 + 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix index f06db98a8efe..e7ddede2ffa4 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix @@ -15,8 +15,7 @@ rec { executable = true; }; - # Packaged resources required for the first bootstrapping stage. - # Contains source code and 256-byte hex0 binary seed. + # Packaged source files for the first bootstrapping stage. # # We don't have access to utilities such as fetchgit and fetchzip since this # is this is part of the bootstrap process and would introduce a circular @@ -25,10 +24,25 @@ rec { # tarballs to be repackaged. # # To build see `make-bootstrap-sources.nix` + + # + # Files came from this Hydra build: + # + # https://hydra.nixos.org/build/ + # + # Which used nixpkgs revision + # to instantiate: + # + # /nix/store/.drv + # + # and then built: + # + # /nix/store/ + # src = import { inherit name; - url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-02/${name}.nar.xz"; - hash = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg="; + url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-18/${name}.nar.xz"; + hash = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U="; unpack = true; }; } diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix index 43859b966add..b85b2f2cac14 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix @@ -9,7 +9,7 @@ derivationWithMeta { pname = "hex0"; builder = hex0-seed; args = [ - "${src}/bootstrap-seeds/POSIX/x86/hex0_x86.hex0" + "${src}/x86/hex0_x86.hex0" (placeholder "out") ]; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix index e85efbbb0243..24fc77f8d345 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix @@ -9,7 +9,7 @@ derivationWithMeta { pname = "kaem-minimal"; builder = hex0; args = [ - "${src}/bootstrap-seeds/POSIX/x86/kaem-minimal.hex0" + "${src}/x86/kaem-minimal.hex0" (placeholder "out") ]; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix index fec03f370bc3..be0974edf430 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix @@ -1,5 +1,4 @@ -# Packaged resources required for the first bootstrapping stage. -# Contains source code and 256-byte hex0 binary seed. +# Packaged source files for the first bootstrapping stage. # # We don't have access to utilities such as fetchgit and fetchzip since this # is this is part of the bootstrap process and would introduce a circular @@ -9,23 +8,30 @@ # # To build: # -# nix-build pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix +# nix-build . -A minimal-bootstrap-sources # => ./result/stage0-posix-$version-$rev-source.nar.xz # -{ pkgs ? import ../../../../.. {} }: +{ lib +, fetchFromGitHub +, runCommand +, nix +, xz +}: let - inherit (pkgs) callPackage runCommand fetchFromGitHub nix xz; - inherit (import ./bootstrap-sources.nix) name rev; src = fetchFromGitHub { owner = "oriansj"; repo = "stage0-posix"; inherit rev; - sha256 = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg="; + sha256 = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U="; fetchSubmodules = true; postFetch = '' + # Seed binaries will be fetched separately + echo "Removing seed binaries" + rm -rf $out/bootstrap-seeds/* + # Remove vendored/duplicate M2libc's echo "Removing duplicate M2libc" rm -rf \ @@ -40,6 +46,14 @@ runCommand name { nativeBuildInputs = [ nix xz ]; passthru = { inherit src; }; + + meta = with lib; { + description = "Packaged sources for the first bootstrapping stage"; + homepage = "https://github.com/oriansj/stage0-posix"; + license = licenses.gpl3Plus; + maintainers = teams.minimal-bootstrap.members; + platforms = platforms.all; + }; } '' mkdir $out nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz" diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 607d824f4faa..761b4fdd0df2 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -27441,6 +27441,7 @@ with pkgs; }; checkMeta = callPackage ../stdenv/generic/check-meta.nix { }; }); + minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { }; mingetty = callPackage ../os-specific/linux/mingetty { }; From df720da7f09899dada20872600a5e7de31a2a995 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 17 Jun 2023 20:57:55 -0700 Subject: [PATCH 2/3] minimal-bootstrap: make sources a non-tarballs.nixos.org FOD This commit adjusts #232576 to break the fetchurl<->minimal-bootstrap-sources dependency cycle without needing an upload to tarballs.nixos.org. It does this by appending the low-level FOD attributes onto the `runCommand` derivation. https://nixos.org/manual/nix/unstable/language/advanced-attributes.html#adv-attr-outputHash --- .../linux/minimal-bootstrap/default.nix | 3 + .../stage0-posix/bootstrap-sources.nix | 93 ++++++++++++------- .../stage0-posix/default.nix | 5 +- .../stage0-posix/make-bootstrap-sources.nix | 11 +-- pkgs/top-level/all-packages.nix | 3 +- 5 files changed, 75 insertions(+), 40 deletions(-) diff --git a/pkgs/os-specific/linux/minimal-bootstrap/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/default.nix index 3e4e9e7f5cb8..3c1746716f06 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/default.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/default.nix @@ -4,6 +4,7 @@ , hostPlatform , fetchurl , checkMeta +, make-minimal-bootstrap-sources }: lib.makeScope @@ -60,6 +61,8 @@ lib.makeScope ln-boot = callPackage ./ln-boot { }; + inherit make-minimal-bootstrap-sources; + mes = lib.recurseIntoAttrs (callPackage ./mes { }); mes-libc = callPackage ./mes/libc.nix { }; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix index e7ddede2ffa4..4aa5d32f7498 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix @@ -1,3 +1,6 @@ +{ make-minimal-bootstrap-sources +}: + rec { name = "stage0-posix-${version}-${rev}-source"; # Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c @@ -5,9 +8,6 @@ rec { rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c"; # This 256 byte seed is the only pre-compiled binary in the bootstrap chain. - # While it is included in the stage0-posix source bundle and is synced with - # stage0-posix updates, we have split it out into its own derivation to highlight - # its unique status as a trusted binary seed. hex0-seed = import { name = "hex0-seed-${version}"; url = "https://github.com/oriansj/bootstrap-seeds/raw/b1263ff14a17835f4d12539226208c426ced4fba/POSIX/x86/hex0-seed"; @@ -15,34 +15,63 @@ rec { executable = true; }; - # Packaged source files for the first bootstrapping stage. - # - # We don't have access to utilities such as fetchgit and fetchzip since this - # is this is part of the bootstrap process and would introduce a circular - # dependency. The only tool we have to fetch source trees is `import ` - # with the unpack option, taking a NAR file as input. This requires source - # tarballs to be repackaged. - # - # To build see `make-bootstrap-sources.nix` + /* + Since `make-minimal-bootstrap-sources` requires nixpkgs and nix it + will create a circular dependency if it is used in place of the + binary bootstrap-files. To break the circular dependency, + `minimal-bootstrap-sources` extends `make-minimal-bootstrap-sources` + by adding Fixed Output Derivation (FOD) attributes. These cause + the builder to be skipped if the expected output is found (by + its hash) in the store or on a substituter. - # - # Files came from this Hydra build: - # - # https://hydra.nixos.org/build/ - # - # Which used nixpkgs revision - # to instantiate: - # - # /nix/store/.drv - # - # and then built: - # - # /nix/store/ - # - src = import { - inherit name; - url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-18/${name}.nar.xz"; - hash = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U="; - unpack = true; - }; + # How do I update the hash? + + Run the following command: + ``` + nix hash file $(nix build --print-out-paths -f '' make-minimal-bootstrap-sources) + ``` + + # Why do we need this `.nar` archive? + + This archive exists only because of a quirk/limitation of Nix: in + restricted mode the builtin fetchers can download only single + files; they have no way to unpack multi-file archives except for + NAR archives: + + https://github.com/NixOS/nixpkgs/pull/232576#issuecomment-1592415619 + + # Why don't we have to upload this to tarballs.nixos.org like the binary bootstrap-files did? + + Unlike this archive, the binary bootstrap-files contained binaries, + which meant that we had to: + + 1. Make sure they came from a trusted builder (Hydra) + 2. Keep careful track of exactly what toolchain (i.e. nixpkgs + commit) that builder used to create them. + 3. Keep copies of the built binaries, in case the toolchains that + produced them failed to be perfectly deterministic. + + The curated archives at tarballs.nixos.org exist in order to + satisfy these requirements. + + The second point created a significant burden: since the nixpkgs + toolchain used to build a given copy of the binary bootstrap-files + itself used a *previous* copy of the bootstrap-files, this meant + we had to track the provenance of all bootstrap-files tarballs + ever used, for all eternity. There was no explanation of where + the "original" bootstrap-files came from: turtles all the way + down. In spite of all this effort we still can't be sure of our + ability to reproduce the binary bootstrap-files, since the + compilers that built them don't always produce exactly bit-for-bit + deterministic results. + + Since this archive contains no binaries and uses a format (NAR) + specifically designed for bit-exact reproducibility, none of the + requirements above apply to `minimal-bootstrap-sources`. + */ + minimal-bootstrap-sources = make-minimal-bootstrap-sources.overrideAttrs(_: { + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg="; + }); } diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix index 2b41261add3a..8e0900100aed 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix @@ -3,7 +3,10 @@ }: lib.makeScope newScope (self: with self; { - inherit (import ./bootstrap-sources.nix) version hex0-seed src; + inherit (self.callPackage ./bootstrap-sources.nix) + version hex0-seed minimal-bootstrap-sources; + + src = minimal-bootstrap-sources; m2libc = src + "/M2libc"; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix index be0974edf430..4fe9a9be7881 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix @@ -8,8 +8,7 @@ # # To build: # -# nix-build . -A minimal-bootstrap-sources -# => ./result/stage0-posix-$version-$rev-source.nar.xz +# nix-build '' -o sources.nar.xz -A make-minimal-bootstrap-sources # { lib @@ -19,7 +18,7 @@ , xz }: let - inherit (import ./bootstrap-sources.nix) name rev; + inherit (import ./bootstrap-sources.nix { make-minimal-bootstrap-sources = null; }) name rev; src = fetchFromGitHub { owner = "oriansj"; @@ -41,8 +40,9 @@ let $out/mescc-tools-extra/M2libc ''; }; + in -runCommand name { +runCommand "${name}.nar.xz" { nativeBuildInputs = [ nix xz ]; passthru = { inherit src; }; @@ -55,6 +55,5 @@ runCommand name { platforms = platforms.all; }; } '' - mkdir $out - nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz" + nix-store --dump ${src} | xz -c > $out '' diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 761b4fdd0df2..55d5131553ca 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -27440,8 +27440,9 @@ with pkgs; inherit (stdenv.buildPlatform) system; }; checkMeta = callPackage ../stdenv/generic/check-meta.nix { }; + inherit make-minimal-bootstrap-sources; }); - minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { }; + make-minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { }; mingetty = callPackage ../os-specific/linux/mingetty { }; From c8080b06a9458f8c898ea4537dab82d32e71d06c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Mon, 26 Jun 2023 00:52:22 -0700 Subject: [PATCH 3/3] separate minimal-bootstrap-sources from make-minimal-bootstrap-sources Ericson2314 observed that although FODs break the build-time cycle, they don't break the eval-time cycle. This cycle must be broken in order to compute the derivation hash. make-minimal-bootstrap-sources does and always has depended on the full nixpkgs (including fetchFromGitHub). This commit completely separates minimal-bootstrap-sources from it, so that minimal-bootstrap-sources now does not depend on make-minimal-bootstrap-sources (or nixpkgs) in any way. minimal-bootstrap-sources is now a "bare Nix" derivation. It is an FOD whose builder always fails with an error message. --- .../linux/minimal-bootstrap/default.nix | 3 -- .../stage0-posix/bootstrap-sources.nix | 37 +++++++++++++++++-- .../stage0-posix/default.nix | 2 +- .../stage0-posix/make-bootstrap-sources.nix | 2 +- pkgs/top-level/all-packages.nix | 2 +- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/pkgs/os-specific/linux/minimal-bootstrap/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/default.nix index 3c1746716f06..3e4e9e7f5cb8 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/default.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/default.nix @@ -4,7 +4,6 @@ , hostPlatform , fetchurl , checkMeta -, make-minimal-bootstrap-sources }: lib.makeScope @@ -61,8 +60,6 @@ lib.makeScope ln-boot = callPackage ./ln-boot { }; - inherit make-minimal-bootstrap-sources; - mes = lib.recurseIntoAttrs (callPackage ./mes { }); mes-libc = callPackage ./mes/libc.nix { }; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix index 4aa5d32f7498..65b052c44569 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix @@ -1,4 +1,4 @@ -{ make-minimal-bootstrap-sources +{ }: rec { @@ -6,6 +6,7 @@ rec { # Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c version = "unstable-2023-05-02"; rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c"; + outputHashAlgo = "sha256"; # This 256 byte seed is the only pre-compiled binary in the bootstrap chain. hex0-seed = import { @@ -69,9 +70,37 @@ rec { specifically designed for bit-exact reproducibility, none of the requirements above apply to `minimal-bootstrap-sources`. */ - minimal-bootstrap-sources = make-minimal-bootstrap-sources.overrideAttrs(_: { + minimal-bootstrap-sources = derivation { + name = "${name}.nar.xz"; + system = builtins.currentSystem; outputHashMode = "flat"; - outputHashAlgo = "sha256"; + inherit outputHashAlgo; outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg="; - }); + + # This builder always fails, but fortunately Nix will print the + # "builder", which is really the error message that we want the + # user to see. + builder = '' + # + # + # Neither your store nor your substituters seems to have: + # + # ${name}.nar.xz + # + # Please obtain or create this file, give it exactly the name + # shown above, and then run the following command: + # + # nix-store --add-fixed ${outputHashAlgo} ${name}.nar.xz + # + # You can create this file from an already-bootstrapped nixpkgs + # using the following command: + # + # nix-build '' -A make-minimal-bootstrap-sources + # + # Or, if you prefer, you can create this file using only `git`, + # `nix`, and `xz`. For the commands needed in order to do this, + # see `make-bootstrap-sources.nix`. + # + ''; + }; } diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix index 8e0900100aed..c15223a43d48 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix @@ -3,7 +3,7 @@ }: lib.makeScope newScope (self: with self; { - inherit (self.callPackage ./bootstrap-sources.nix) + inherit (self.callPackage ./bootstrap-sources.nix {}) version hex0-seed minimal-bootstrap-sources; src = minimal-bootstrap-sources; diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix index 4fe9a9be7881..8951cc9ddbc4 100644 --- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix +++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix @@ -18,7 +18,7 @@ , xz }: let - inherit (import ./bootstrap-sources.nix { make-minimal-bootstrap-sources = null; }) name rev; + inherit (import ./bootstrap-sources.nix { }) name rev; src = fetchFromGitHub { owner = "oriansj"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 55d5131553ca..e68f4669541f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -27440,8 +27440,8 @@ with pkgs; inherit (stdenv.buildPlatform) system; }; checkMeta = callPackage ../stdenv/generic/check-meta.nix { }; - inherit make-minimal-bootstrap-sources; }); + minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix { }; make-minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { }; mingetty = callPackage ../os-specific/linux/mingetty { };