Merge pull request #238357 from amjoseph-nixpkgs/review/232576

minimal-bootstrap: make sources a non-tarballs.nixos.org FOD

pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix

@@ -1,13 +1,14 @@
+{
+}:
+
 rec {
   name = "stage0-posix-${version}-${rev}-source";
   # Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c
   version = "unstable-2023-05-02";
   rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c";
+  outputHashAlgo = "sha256";
 
   # This 256 byte seed is the only pre-compiled binary in the bootstrap chain.
-  # While it is included in the stage0-posix source bundle and is synced with
-  # stage0-posix updates, we have split it out into its own derivation to highlight
-  # its unique status as a trusted binary seed.
   hex0-seed = import <nix/fetchurl.nix> {
     name = "hex0-seed-${version}";
     url = "https://github.com/oriansj/bootstrap-seeds/raw/b1263ff14a17835f4d12539226208c426ced4fba/POSIX/x86/hex0-seed";
@@ -15,20 +16,91 @@ rec {
     executable = true;
   };
 
-  # Packaged resources required for the first bootstrapping stage.
-  # Contains source code and 256-byte hex0 binary seed.
+  /*
+  Since `make-minimal-bootstrap-sources` requires nixpkgs and nix it
+  will create a circular dependency if it is used in place of the
+  binary bootstrap-files.  To break the circular dependency,
+  `minimal-bootstrap-sources` extends `make-minimal-bootstrap-sources`
+  by adding Fixed Output Derivation (FOD) attributes.  These cause
+  the builder to be skipped if the expected output is found (by
+  its hash) in the store or on a substituter.
+
+  # How do I update the hash?
+
+  Run the following command:
+  ```
+  nix hash file $(nix build --print-out-paths -f '<nixpkgs>' make-minimal-bootstrap-sources)
+  ```
+
+  # Why do we need this `.nar` archive?
+
+  This archive exists only because of a quirk/limitation of Nix: in
+  restricted mode the builtin fetchers can download only single
+  files; they have no way to unpack multi-file archives except for
+  NAR archives:
+
+  https://github.com/NixOS/nixpkgs/pull/232576#issuecomment-1592415619
+
+  # Why don't we have to upload this to tarballs.nixos.org like the binary bootstrap-files did?
+
+  Unlike this archive, the binary bootstrap-files contained binaries,
+  which meant that we had to:
+
+  1. Make sure they came from a trusted builder (Hydra)
+  2. Keep careful track of exactly what toolchain (i.e. nixpkgs
+     commit) that builder used to create them.
+  3. Keep copies of the built binaries, in case the toolchains that
+     produced them failed to be perfectly deterministic.
+
+  The curated archives at tarballs.nixos.org exist in order to
+  satisfy these requirements.
+
+  The second point created a significant burden: since the nixpkgs
+  toolchain used to build a given copy of the binary bootstrap-files
+  itself used a *previous* copy of the bootstrap-files, this meant
+  we had to track the provenance of all bootstrap-files tarballs
+  ever used, for all eternity.  There was no explanation of where
+  the "original" bootstrap-files came from: turtles all the way
+  down.  In spite of all this effort we still can't be sure of our
+  ability to reproduce the binary bootstrap-files, since the
+  compilers that built them don't always produce exactly bit-for-bit
+  deterministic results.
+
+  Since this archive contains no binaries and uses a format (NAR)
+  specifically designed for bit-exact reproducibility, none of the
+  requirements above apply to `minimal-bootstrap-sources`.
+  */
+  minimal-bootstrap-sources = derivation {
+    name = "${name}.nar.xz";
+    system = builtins.currentSystem;
+    outputHashMode = "flat";
+    inherit outputHashAlgo;
+    outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg=";
+
+    # This builder always fails, but fortunately Nix will print the
+    # "builder", which is really the error message that we want the
+    # user to see.
+    builder = ''
       #
-  # We don't have access to utilities such as fetchgit and fetchzip since this
-  # is this is part of the bootstrap process and would introduce a circular
-  # dependency. The only tool we have to fetch source trees is `import <nix/fetchurl.nix>`
-  # with the unpack option, taking a NAR file as input. This requires source
-  # tarballs to be repackaged.
       #
-  # To build see `make-bootstrap-sources.nix`
-  src = import <nix/fetchurl.nix> {
-    inherit name;
-    url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-02/${name}.nar.xz";
-    hash = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
-    unpack = true;
+      # Neither your store nor your substituters seems to have:
+      #
+      #  ${name}.nar.xz
+      #
+      # Please obtain or create this file, give it exactly the name
+      # shown above, and then run the following command:
+      #
+      #   nix-store --add-fixed ${outputHashAlgo} ${name}.nar.xz
+      #
+      # You can create this file from an already-bootstrapped nixpkgs
+      # using the following command:
+      #
+      #   nix-build '<nixpkgs>' -A make-minimal-bootstrap-sources
+      #
+      # Or, if you prefer, you can create this file using only `git`,
+      # `nix`, and `xz`.  For the commands needed in order to do this,
+      # see `make-bootstrap-sources.nix`.
+      #
+    '';
   };
 }

pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix

@@ -3,7 +3,10 @@
 }:
 
 lib.makeScope newScope (self: with self; {
-  inherit (import ./bootstrap-sources.nix) version hex0-seed src;
+  inherit (self.callPackage ./bootstrap-sources.nix {})
+    version hex0-seed minimal-bootstrap-sources;
+
+  src = minimal-bootstrap-sources;
 
   m2libc = src + "/M2libc";
 

pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix

@@ -9,7 +9,7 @@ derivationWithMeta {
   pname = "hex0";
   builder = hex0-seed;
   args = [
-    "${src}/bootstrap-seeds/POSIX/x86/hex0_x86.hex0"
+    "${src}/x86/hex0_x86.hex0"
     (placeholder "out")
   ];
 

pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix

@@ -9,7 +9,7 @@ derivationWithMeta {
   pname = "kaem-minimal";
   builder = hex0;
   args = [
-    "${src}/bootstrap-seeds/POSIX/x86/kaem-minimal.hex0"
+    "${src}/x86/kaem-minimal.hex0"
     (placeholder "out")
   ];
 

pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix

@@ -1,5 +1,4 @@
-# Packaged resources required for the first bootstrapping stage.
-# Contains source code and 256-byte hex0 binary seed.
+# Packaged source files for the first bootstrapping stage.
 #
 # We don't have access to utilities such as fetchgit and fetchzip since this
 # is this is part of the bootstrap process and would introduce a circular
@@ -9,23 +8,29 @@
 #
 # To build:
 #
-#   nix-build pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
-#   => ./result/stage0-posix-$version-$rev-source.nar.xz
+#   nix-build '<nixpkgs>' -o sources.nar.xz -A make-minimal-bootstrap-sources
 #
 
-{ pkgs ? import ../../../../.. {} }:
+{ lib
+, fetchFromGitHub
+, runCommand
+, nix
+, xz
+}:
 let
-  inherit (pkgs) callPackage runCommand fetchFromGitHub nix xz;
-
-  inherit (import ./bootstrap-sources.nix) name rev;
+  inherit (import ./bootstrap-sources.nix { }) name rev;
 
   src = fetchFromGitHub {
     owner = "oriansj";
     repo = "stage0-posix";
     inherit rev;
-    sha256 = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
+    sha256 = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U=";
     fetchSubmodules = true;
     postFetch = ''
+      # Seed binaries will be fetched separately
+      echo "Removing seed binaries"
+      rm -rf $out/bootstrap-seeds/*
+
       # Remove vendored/duplicate M2libc's
       echo "Removing duplicate M2libc"
       rm -rf \
@@ -35,12 +40,20 @@ let
         $out/mescc-tools-extra/M2libc
     '';
   };
+
 in
-runCommand name {
+runCommand "${name}.nar.xz" {
   nativeBuildInputs = [ nix xz ];
 
   passthru = { inherit src; };
+
+  meta = with lib; {
+    description = "Packaged sources for the first bootstrapping stage";
+    homepage = "https://github.com/oriansj/stage0-posix";
+    license = licenses.gpl3Plus;
+    maintainers = teams.minimal-bootstrap.members;
+    platforms = platforms.all;
+  };
 } ''
-  mkdir $out
-  nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz"
+  nix-store --dump ${src} | xz -c > $out
 ''

pkgs/top-level/all-packages.nix

@@ -27559,6 +27559,8 @@ with pkgs;
     };
     checkMeta = callPackage ../stdenv/generic/check-meta.nix { };
   });
+  minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix { };
+  make-minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { };
 
   mingetty = callPackage ../os-specific/linux/mingetty { };