From 2a8d05627bc98b25d65244bb208583ac0b5c3007 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 8 May 2020 20:48:30 -0500 Subject: [PATCH 1/3] =?UTF-8?q?build-bazel-package:=20switch=20hash=20mode?= =?UTF-8?q?=20to=20=E2=80=9Cflat=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit flat hashes can be substituted through hashed-mirrors, while recursive hashes can’t. This is especially important for Bazel since the bazel fetch dependencies can come from multiple different methods (git, http, ftp, etc.). To do this, we create tar archives from the output/external directory, which is then extracted to build. All of the Bazel hashes are all updated. --- .../applications/virtualization/gvisor/default.nix | 2 +- pkgs/build-support/build-bazel-package/default.nix | 14 +++++++------- .../python-modules/dm-sonnet/default.nix | 2 +- .../tensorflow-probability/default.nix | 2 +- .../python-modules/tensorflow/1/default.nix | 4 ++-- .../python-modules/tensorflow/2/default.nix | 4 ++-- pkgs/development/tools/bazel-watcher/default.nix | 2 +- .../build-managers/bazel/bazel-remote/default.nix | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkgs/applications/virtualization/gvisor/default.nix b/pkgs/applications/virtualization/gvisor/default.nix index e157ca825af0..7d62b1b9fa24 100644 --- a/pkgs/applications/virtualization/gvisor/default.nix +++ b/pkgs/applications/virtualization/gvisor/default.nix @@ -76,7 +76,7 @@ in buildBazelPackage rec { rm -f "$bazelOut"/java.log "$bazelOut"/java.log.* ''; - sha256 = "1bn7nhv5pag8fdm8l8nvgg3fzvhpy2yv9yl2slrb16lckxzha3v6"; + sha256 = "0r11kbyp1ambgcj35gvjjmxrsrdg7b9jb9sq3kih4lik7zyljp25"; }; buildAttrs = { diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 9a64b84f076b..61befff3eaaa 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -42,7 +42,7 @@ in stdenv.mkDerivation (fBuildAttrs // { inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget; deps = stdenv.mkDerivation (fFetchAttrs // { - name = "${name}-deps"; + name = "${name}-deps.tar.gz"; inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget; nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ]; @@ -120,9 +120,9 @@ in stdenv.mkDerivation (fBuildAttrs // { ln -sf "$new_target" "$symlink" done - cp -r $bazelOut/external $out + echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version - echo '${bazel.name}' > $out/.nix-bazel-version + (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/) runHook postInstall ''; @@ -130,7 +130,6 @@ in stdenv.mkDerivation (fBuildAttrs // { dontFixup = true; allowedRequisites = []; - outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = fetchAttrs.sha256; }); @@ -146,14 +145,15 @@ in stdenv.mkDerivation (fBuildAttrs // { preConfigure = '' mkdir -p "$bazelOut" - test "${bazel.name}" = "$(<$deps/.nix-bazel-version)" || { + (cd $bazelOut && tar xfz $deps) + + test "${bazel.name}" = "$(<$bazelOut/external/.nix-bazel-version)" || { echo "fixed output derivation was built for a different bazel version" >&2 - echo " got: $(<$deps/.nix-bazel-version)" >&2 + echo " got: $(<$bazelOut/external/.nix-bazel-version)" >&2 echo "expected: ${bazel.name}" >&2 exit 1 } - cp -r $deps $bazelOut/external chmod -R +w $bazelOut find $bazelOut -type l | while read symlink; do ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink" diff --git a/pkgs/development/python-modules/dm-sonnet/default.nix b/pkgs/development/python-modules/dm-sonnet/default.nix index 08dbafe019e4..4205d32d4958 100644 --- a/pkgs/development/python-modules/dm-sonnet/default.nix +++ b/pkgs/development/python-modules/dm-sonnet/default.nix @@ -36,7 +36,7 @@ let bazelTarget = ":install"; fetchAttrs = { - sha256 = "0wb2gh9ji8bgq4s9ci9x017dybxqzjhncpw33b1wjksm2yhbkvlz"; + sha256 = "0jlgmwcanfkhkwvh4vxibpi0zyyl6xkjzcz2xqn7raympqscq81a"; }; bazelFlags = [ diff --git a/pkgs/development/python-modules/tensorflow-probability/default.nix b/pkgs/development/python-modules/tensorflow-probability/default.nix index 3a5ee229b432..9ae3464f3f8f 100644 --- a/pkgs/development/python-modules/tensorflow-probability/default.nix +++ b/pkgs/development/python-modules/tensorflow-probability/default.nix @@ -61,7 +61,7 @@ let bazelTarget = ":pip_pkg"; fetchAttrs = { - sha256 = "0135nxxvkmjzpd80r1g9fdkk9h62g0xlvp32g5zgk0hkma5kq0bx"; + sha256 = "0nmk7sbgl1ag20333v9r7l5cka6wapbskikfhi3y5a8l9f6hxkl3"; }; buildAttrs = { diff --git a/pkgs/development/python-modules/tensorflow/1/default.nix b/pkgs/development/python-modules/tensorflow/1/default.nix index aae812e8ebeb..7c628a0789b0 100644 --- a/pkgs/development/python-modules/tensorflow/1/default.nix +++ b/pkgs/development/python-modules/tensorflow/1/default.nix @@ -300,9 +300,9 @@ let # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "1qygfcvvn9vysap9nk6xccxi9mgmzyxiywz6k456f811l1v70p2c" + "09j57w6kc0vkfcdwr0qggy3qgrgq82kfa2jrwvvcnij4bl3wj40j" else - "0kfjanw0mfbh30vi1ms2xlg8yp429cbyfriik6yxd5cla2pncg2j"; + "14g8z49qz7d8n1c2mcsfhr7yqpcy7mhmpm6hgmqvpgb8vm7yvwrc"; }; buildAttrs = { diff --git a/pkgs/development/python-modules/tensorflow/2/default.nix b/pkgs/development/python-modules/tensorflow/2/default.nix index 089d377ec241..3b4f45f3176f 100644 --- a/pkgs/development/python-modules/tensorflow/2/default.nix +++ b/pkgs/development/python-modules/tensorflow/2/default.nix @@ -294,9 +294,9 @@ let # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "0hg3ysy644950a34j28hrb317cz8gcbb9n84d36wdailvnlshghb" + "1kqk1gx5g63kb2zdj392x6mnpbrmgqghrdv597aipn7s23xzj8pd" else - "1gy4pz9kn30wb9c4a9584fibb88c3h38y3dqa99yw1lbsbyyi28c"; + "1plpcm2ydpajsrxdvmmpfy7l0gfdir78hap72w4k7ddm6d3rm2fv"; }; buildAttrs = { diff --git a/pkgs/development/tools/bazel-watcher/default.nix b/pkgs/development/tools/bazel-watcher/default.nix index 9cdd72eeaaf4..2d9b99a35f86 100644 --- a/pkgs/development/tools/bazel-watcher/default.nix +++ b/pkgs/development/tools/bazel-watcher/default.nix @@ -60,7 +60,7 @@ buildBazelPackage rec { sed -e '/^FILE:@bazel_gazelle_go_repository_tools.*/d' -i $bazelOut/external/\@*.marker ''; - sha256 = "0wj573dcirssr2cmq90b4yl57mv3gsxaj1s26afvkz1dvyxmq7sz"; + sha256 = "0ili0123xcl0mjcrn4r1r1q9d7a8a3bbh9r3zhlshl39awfm4r2d"; }; buildAttrs = { diff --git a/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix b/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix index f6d112800b68..6b8269584bd2 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix @@ -64,7 +64,7 @@ buildBazelPackage rec { sed -e '/^FILE:@bazel_gazelle_go_repository_tools.*/d' -i $bazelOut/external/\@*.marker ''; - sha256 = "141kw2zpr612xdcrg6x9kslg4d5b3fbpzx0vgp3lqwdihfj3sc1l"; + sha256 = "1hzb84zxav68ysdwkimaaz0b9rq95k3bgifw01m644v7idabrz50"; }; buildAttrs = { From 1e19cc37bd7b3d137cd738494eec4116b6585574 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 9 May 2020 13:35:47 -0500 Subject: [PATCH 2/3] build-bazel-package: pass proxy env vars --- pkgs/build-support/build-bazel-package/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 61befff3eaaa..31c0c8e92e56 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -45,6 +45,8 @@ in stdenv.mkDerivation (fBuildAttrs // { name = "${name}-deps.tar.gz"; inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ]; preHook = fFetchAttrs.preHook or "" + '' From fe48f63c3ce022ff190c14ea6bf7a2fffd0ca102 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 9 May 2020 14:13:23 -0500 Subject: [PATCH 3/3] build-bazel-package: Add hash change to changelog --- nixos/doc/manual/release-notes/rl-2009.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2009.xml b/nixos/doc/manual/release-notes/rl-2009.xml index 744be530be75..eeb290b606e4 100644 --- a/nixos/doc/manual/release-notes/rl-2009.xml +++ b/nixos/doc/manual/release-notes/rl-2009.xml @@ -326,6 +326,15 @@ php.override { your configuration simply remove the quotes around the numbers. + + + When using buildBazelPackage from Nixpkgs, + flat hash mode is now used for dependencies + instead of recursive. This is to better allow + using hashed mirrors where needed. As a result, these hashes + will have changed. + +