maintainers/scripts/rebuild-amount.sh: report parallelism, add example, cleanup (#137695)
* maintainers/scripts/rebuild-amount.sh: report parallelism setting This slightly helps discovering parallelism support. * maintainers/scripts/rebuild-amount.sh: add basic example It's not clear if script should be used against HEAD or HEAD^. Let's have a copyable example. * maintainers/scripts/rebuild-amount.sh: convert tabs to four spaces One notable change is here-document conversion. Before: <TAB>cat <<-FOO <TAB>... <TAB>FOO After: <sp><sp><sp><sp>cat <<FOO <sp><sp><sp><sp>... FOO Note seemingly misaligned 'FOO': '-FOO' understands leading tabs, but not spaces.
This commit is contained in:
Sergei Trofimovich
GitHub
parent
1922e87924
commit
39e8ec2db6
@ -4,123 +4,130 @@ set -e
|
||||
# --print: avoid dependency on environment
|
||||
optPrint=
|
||||
if [ "$1" == "--print" ]; then
|
||||
optPrint=true
|
||||
shift
|
||||
optPrint=true
|
||||
shift
|
||||
fi
|
||||
|
||||
if [ "$#" != 1 ] && [ "$#" != 2 ]; then
|
||||
cat <<-EOF
|
||||
Usage: $0 [--print] commit-spec [commit-spec]
|
||||
You need to be in a git-controlled nixpkgs tree.
|
||||
The current state of the tree will be used if the second commit is missing.
|
||||
EOF
|
||||
exit 1
|
||||
cat <<EOF
|
||||
Usage: $0 [--print] from-commit-spec [to-commit-spec]
|
||||
You need to be in a git-controlled nixpkgs tree.
|
||||
The current state of the tree will be used if the second commit is missing.
|
||||
|
||||
Examples:
|
||||
effect of latest commit:
|
||||
$ $0 HEAD^
|
||||
$ $0 --print HEAD^
|
||||
effect of the whole patch series for 'staging' branch:
|
||||
$ $0 origin/staging staging
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A slightly hacky way to get the config.
|
||||
parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
|
||||
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
|
||||
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
|
||||
|
||||
echo "Estimating rebuild amount by counting changed Hydra jobs."
|
||||
echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})."
|
||||
|
||||
toRemove=()
|
||||
|
||||
cleanup() {
|
||||
rm -rf "${toRemove[@]}"
|
||||
rm -rf "${toRemove[@]}"
|
||||
}
|
||||
trap cleanup EXIT SIGINT SIGQUIT ERR
|
||||
|
||||
MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
|
||||
|
||||
nixexpr() {
|
||||
cat <<-EONIX
|
||||
let
|
||||
lib = import $1/lib;
|
||||
hydraJobs = import $1/pkgs/top-level/release.nix
|
||||
# Compromise: accuracy vs. resources needed for evaluation.
|
||||
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
|
||||
cfg = (import $1 {}).config.rebuild-amount or {};
|
||||
cat <<EONIX
|
||||
let
|
||||
lib = import $1/lib;
|
||||
hydraJobs = import $1/pkgs/top-level/release.nix
|
||||
# Compromise: accuracy vs. resources needed for evaluation.
|
||||
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
|
||||
cfg = (import $1 {}).config.rebuild-amount or {};
|
||||
|
||||
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
|
||||
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
|
||||
|
||||
# hydraJobs leaves recurseForDerivations as empty attrmaps;
|
||||
# that would break nix-env and we also need to recurse everywhere.
|
||||
tweak = lib.mapAttrs
|
||||
(name: val:
|
||||
if name == "recurseForDerivations" then true
|
||||
else if lib.isAttrs val && val.type or null != "derivation"
|
||||
then recurseIntoAttrs (tweak val)
|
||||
else val
|
||||
);
|
||||
# hydraJobs leaves recurseForDerivations as empty attrmaps;
|
||||
# that would break nix-env and we also need to recurse everywhere.
|
||||
tweak = lib.mapAttrs
|
||||
(name: val:
|
||||
if name == "recurseForDerivations" then true
|
||||
else if lib.isAttrs val && val.type or null != "derivation"
|
||||
then recurseIntoAttrs (tweak val)
|
||||
else val
|
||||
);
|
||||
|
||||
# Some of these contain explicit references to platform(s) we want to avoid;
|
||||
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
|
||||
blacklist = [
|
||||
"tarball" "metrics" "manual"
|
||||
"darwin-tested" "unstable" "stdenvBootstrapTools"
|
||||
"moduleSystem" "lib-tests" # these just confuse the output
|
||||
];
|
||||
# Some of these contain explicit references to platform(s) we want to avoid;
|
||||
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
|
||||
blacklist = [
|
||||
"tarball" "metrics" "manual"
|
||||
"darwin-tested" "unstable" "stdenvBootstrapTools"
|
||||
"moduleSystem" "lib-tests" # these just confuse the output
|
||||
];
|
||||
|
||||
in
|
||||
tweak (builtins.removeAttrs hydraJobs blacklist)
|
||||
EONIX
|
||||
in
|
||||
tweak (builtins.removeAttrs hydraJobs blacklist)
|
||||
EONIX
|
||||
}
|
||||
|
||||
# Output packages in tree $2 that weren't in $1.
|
||||
# Changing the output hash or name is taken as a change.
|
||||
# Extra nix-env parameters can be in $3
|
||||
newPkgs() {
|
||||
# We use files instead of pipes, as running multiple nix-env processes
|
||||
# could eat too much memory for a standard 4GiB machine.
|
||||
local -a list
|
||||
for i in 1 2; do
|
||||
local l="$($MKTEMP)"
|
||||
list[$i]="$l"
|
||||
toRemove+=("$l")
|
||||
# We use files instead of pipes, as running multiple nix-env processes
|
||||
# could eat too much memory for a standard 4GiB machine.
|
||||
local -a list
|
||||
for i in 1 2; do
|
||||
local l="$($MKTEMP)"
|
||||
list[$i]="$l"
|
||||
toRemove+=("$l")
|
||||
|
||||
local expr="$($MKTEMP)"
|
||||
toRemove+=("$expr")
|
||||
nixexpr "${!i}" > "$expr"
|
||||
local expr="$($MKTEMP)"
|
||||
toRemove+=("$expr")
|
||||
nixexpr "${!i}" > "$expr"
|
||||
|
||||
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
|
||||
| sort > "${list[$i]}" &
|
||||
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
|
||||
| sort > "${list[$i]}" &
|
||||
|
||||
if [ "$parallel" != "true" ]; then
|
||||
wait
|
||||
fi
|
||||
done
|
||||
if [ "$parallel" != "true" ]; then
|
||||
wait
|
||||
fi
|
||||
done
|
||||
|
||||
wait
|
||||
comm -13 "${list[@]}"
|
||||
wait
|
||||
comm -13 "${list[@]}"
|
||||
}
|
||||
|
||||
# Prepare nixpkgs trees.
|
||||
declare -a tree
|
||||
for i in 1 2; do
|
||||
if [ -n "${!i}" ]; then # use the given commit
|
||||
dir="$($MKTEMP -d)"
|
||||
tree[$i]="$dir"
|
||||
toRemove+=("$dir")
|
||||
if [ -n "${!i}" ]; then # use the given commit
|
||||
dir="$($MKTEMP -d)"
|
||||
tree[$i]="$dir"
|
||||
toRemove+=("$dir")
|
||||
|
||||
git clone --shared --no-checkout --quiet . "${tree[$i]}"
|
||||
(cd "${tree[$i]}" && git checkout --quiet "${!i}")
|
||||
else #use the current tree
|
||||
tree[$i]="$(pwd)"
|
||||
fi
|
||||
git clone --shared --no-checkout --quiet . "${tree[$i]}"
|
||||
(cd "${tree[$i]}" && git checkout --quiet "${!i}")
|
||||
else #use the current tree
|
||||
tree[$i]="$(pwd)"
|
||||
fi
|
||||
done
|
||||
|
||||
newlist="$($MKTEMP)"
|
||||
toRemove+=("$newlist")
|
||||
# Notes:
|
||||
# - the evaluation is done on x86_64-linux, like on Hydra.
|
||||
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
|
||||
# - the evaluation is done on x86_64-linux, like on Hydra.
|
||||
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
|
||||
newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
|
||||
|
||||
# Hacky: keep only the last word of each attribute path and sort.
|
||||
sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
|
||||
| sort | uniq -c
|
||||
| sort | uniq -c
|
||||
|
||||
if [ -n "$optPrint" ]; then
|
||||
echo
|
||||
cat "$newlist"
|
||||
echo
|
||||
cat "$newlist"
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user