diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps index 5074e26848ed..4fbc53d1039e 100755 --- a/pkgs/build-support/rust/fetch-cargo-deps +++ b/pkgs/build-support/rust/fetch-cargo-deps @@ -1,183 +1,177 @@ -source $stdenv/setup +fetchCargoDeps() { + src=$(realpath $1) + out=$(realpath $2) -set -euo pipefail + echo "Fetching $src to $out" -src=$(realpath $1) -out=$(realpath $2) + mkdir $out -echo "Fetching $src to $out" + # Configure cargo to fetch from a local copy of the crates.io registry -mkdir $out + echo "Using rust registry from $rustRegistry" -# Configure cargo to fetch from a local copy of the crates.io registry - -echo "Using rust registry from $rustRegistry" - -cat < $out/config + cat < $out/config [registry] index = "file://$rustRegistry" EOF -export CARGO_HOME=$out -cd $src + export CARGO_HOME=$out + cd $src -if [[ ! -f Cargo.lock ]]; then - echo - echo "ERROR: The Cargo.lock file doesn't exist" - echo - echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" - echo "when the registry is updated." - echo + if [[ ! -f Cargo.lock ]]; then + echo + echo "ERROR: The Cargo.lock file doesn't exist" + echo + echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" + echo "when the registry is updated." + echo - exit 1 -fi + exit 1 + fi -# We need to do the following string replacement so that 'cargo fetch' -# doesn't ignore the versions specified in Cargo.lock -set +u -substituteInPlace Cargo.lock \ - --replace "registry+https://github.com/rust-lang/crates.io-index" \ - "registry+file://$rustRegistry" -set -u + # We need to do the following string replacement so that 'cargo fetch' + # doesn't ignore the versions specified in Cargo.lock + substituteInPlace Cargo.lock \ + --replace "registry+https://github.com/rust-lang/crates.io-index" \ + "registry+file://$rustRegistry" -# Do any possible 'cargo update -p --precise ' ad-hoc updates -eval "$cargoUpdateHook" + # Do any possible 'cargo update -p --precise ' ad-hoc updates + eval "$cargoUpdateHook" -# Do the fetch -cargo fetch --verbose + # Do the fetch + cargo fetch --verbose -# Now that we have fetched everything, let's make the output deterministic + # Now that we have fetched everything, let's make the output deterministic -# Cargo uses the following directory structure for fetched data, where -# $indexHash is a hash of the registry index URL: -# -# -# /config: -# -# Cargo config file. We'll delete this because it's not deterministic, -# and instead recreate it just before running 'cargo build'. -# -# /registry/cache/$indexHash/: -# -# This is where tarballs of registry package dependencies are kept -# We'll need to keep this, but make sure $indexHash is a fixed name. -# -# /registry/index/$indexHash/: -# -# A copy of the registry index is kept here. We can delete this, and -# instead, just before running 'cargo build', we'll symlink this -# directory to our static copy of the registry in the Nix store. -# -# /registry/src/$indexHash/{pkgName-pkgVersion}/: -# -# Here cargo keeps extracted sources of the cached tarballs. -# We'll just delete this because cargo will re-populate them from the -# tarballs. -# -# /git/db/{domain-hash}/: -# -# Here cargo keeps the `.git` directories of git dependencies. -# We'll need to keep these, but make them deterministic. -# -# /git/checkouts/{domain-hash}/{branchName}/: -# -# Here cargo keeps checked-out sources of the git dependencies. -# We can delete this, because cargo will re-populate them from the above -# `.git` directories. -# -# Let's start + # Cargo uses the following directory structure for fetched data, where + # $indexHash is a hash of the registry index URL: + # + # + # /config: + # + # Cargo config file. We'll delete this because it's not deterministic, + # and instead recreate it just before running 'cargo build'. + # + # /registry/cache/$indexHash/: + # + # This is where tarballs of registry package dependencies are kept + # We'll need to keep this, but make sure $indexHash is a fixed name. + # + # /registry/index/$indexHash/: + # + # A copy of the registry index is kept here. We can delete this, and + # instead, just before running 'cargo build', we'll symlink this + # directory to our static copy of the registry in the Nix store. + # + # /registry/src/$indexHash/{pkgName-pkgVersion}/: + # + # Here cargo keeps extracted sources of the cached tarballs. + # We'll just delete this because cargo will re-populate them from the + # tarballs. + # + # /git/db/{domain-hash}/: + # + # Here cargo keeps the `.git` directories of git dependencies. + # We'll need to keep these, but make them deterministic. + # + # /git/checkouts/{domain-hash}/{branchName}/: + # + # Here cargo keeps checked-out sources of the git dependencies. + # We can delete this, because cargo will re-populate them from the above + # `.git` directories. + # + # Let's start -# Remove cargo config file, which points to the ever-changing registry -rm $out/config + # Remove cargo config file, which points to the ever-changing registry + rm $out/config -# Save the Cargo.lock file into the output, so that we don't have to do another -# 'cargo update' during the build (which would try to access the network) for -# any ad-hoc package updates (through $cargoUpdateHook). -# -# We need to replace the rustRegistry URL with something deterministic. -# Since the URL won't actually be accessed anymore, it's fine to use /dev/null. + # Save the Cargo.lock file into the output, so that we don't have to do another + # 'cargo update' during the build (which would try to access the network) for + # any ad-hoc package updates (through $cargoUpdateHook). + # + # We need to replace the rustRegistry URL with something deterministic. + # Since the URL won't actually be accessed anymore, it's fine to use /dev/null. -set +u -substituteInPlace Cargo.lock \ - --replace "registry+file://$rustRegistry" \ - "registry+file:///dev/null" -set -u -mv Cargo.lock $out/ + substituteInPlace Cargo.lock \ + --replace "registry+file://$rustRegistry" \ + "registry+file:///dev/null" + mv Cargo.lock $out/ -# Let's replace $indexHash with something more deterministic -mv $out/registry/cache/* $out/registry/cache/HASH + # Let's replace $indexHash with something more deterministic + mv $out/registry/cache/* $out/registry/cache/HASH -# The registry index changes all the time, so it's not deterministic -# We'll symlink it before running 'cargo build' -rm -rf $out/registry/index/* + # The registry index changes all the time, so it's not deterministic + # We'll symlink it before running 'cargo build' + rm -rf $out/registry/index/* -# Make git DBs deterministic -# TODO: test with git submodules -[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do - revs="" - cd "$out/git/checkouts/$name" - while read dir; do - # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] - branch="${dir:2:$((${#dir}-7))}" + # Make git DBs deterministic + # TODO: test with git submodules + [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do + revs="" + cd "$out/git/checkouts/$name" + while read dir; do + # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] + branch="${dir:2:$((${#dir}-7))}" - cd "$out/git/checkouts/$name/$branch" - rev="$(git rev-parse HEAD)" - revs="$revs $rev" - done < <(find . -type d -name .git -print) + cd "$out/git/checkouts/$name/$branch" + rev="$(git rev-parse HEAD)" + revs="$revs $rev" + done < <(find . -type d -name .git -print) - echo "List of revs to keep for git db $name: $revs" + echo "List of revs to keep for git db $name: $revs" - ( - # The following code was adapted from nix-prefetch-git + ( + # The following code was adapted from nix-prefetch-git - cd "$out/git/db/$name" + cd "$out/git/db/$name" - export GIT_DIR=. + export GIT_DIR=. - # Remove all remote branches - git branch -r | while read branch; do - git branch -rD "$branch" >&2 - done + # Remove all remote branches + git branch -r | while read branch; do + git branch -rD "$branch" >&2 + done - # Remove all tags - git tag | while read tag; do - git tag -d "$tag" >&2 - done + # Remove all tags + git tag | while read tag; do + git tag -d "$tag" >&2 + done - # Remove all local branches - branchrefs=() - eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" + # Remove all local branches + branchrefs=() + eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" - for branchref in "${branchrefs[@]}"; do - git update-ref -d "$branchref" >&2 - done + for branchref in "${branchrefs[@]}"; do + git update-ref -d "$branchref" >&2 + done - # Create ad-hoc branches for the revs we need - echo "$revs" | while read rev; do - echo "Creating git branch b_$rev $rev" - git branch b_$rev $rev - done + # Create ad-hoc branches for the revs we need + echo "$revs" | while read rev; do + echo "Creating git branch b_$rev $rev" + git branch b_$rev $rev + done - # Remove files that have timestamps or otherwise have non-deterministic - # properties. - rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config + # Remove files that have timestamps or otherwise have non-deterministic + # properties. + rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config - # Do a full repack. Must run single-threaded, or else we lose determinism. - git config pack.threads 1 - git repack -A -d -f - rm -f config + # Do a full repack. Must run single-threaded, or else we lose determinism. + git config pack.threads 1 + git repack -A -d -f + rm -f config - # Garbage collect unreferenced objects. - git gc --prune=all - ) -done) + # Garbage collect unreferenced objects. + git gc --prune=all + ) + done) -# Remove unneeded outputs -[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src -[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts + # Remove unneeded outputs + [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src + [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts -# XXX: provide some debugging output to see find out why we are seeing -# sporadic hash mismatches -find $out ! -type f -find $out -type f -exec sha256sum {} + + # XXX: provide some debugging output to see find out why we are seeing + # sporadic hash mismatches + find $out ! -type f + find $out -type f -exec sha256sum {} + +} diff --git a/pkgs/build-support/rust/fetchcargo.nix b/pkgs/build-support/rust/fetchcargo.nix index a77540c396f2..518420002622 100644 --- a/pkgs/build-support/rust/fetchcargo.nix +++ b/pkgs/build-support/rust/fetchcargo.nix @@ -9,8 +9,11 @@ stdenv.mkDerivation { phases = "unpackPhase installPhase"; installPhase = '' + source ${./fetch-cargo-deps} + export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt - bash ${./fetch-cargo-deps} . "$out" + + fetchCargoDeps . "$out" ''; outputHashAlgo = "sha256";