From ca0c46040649ab4a6d5d7bc838b393fdcbcae10c Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Thu, 26 Nov 2015 14:44:49 +0100 Subject: [PATCH] nixos-rebuild: Add option for building and/or deploying on a remote host This commit adds the options --build-host and --target-host to nixos-rebuild. --build-host instructs nixos-rebuild to perform all nix builds on the specified host (via ssh). Build results are then copied back to the local machine and used when activating the system. --build-target instructs nixos-rebuild to activate the configuration not on the local machine but on the specified remote host. Build results are copied to the target machine and then activated there (via ssh). It is possible to combine the usage of --build-host and --target-host, in which case you can perform the build on one remote machine and deploy the configuration to another remote machine. The only requirement is that the build host has a working ssh connection to the target host (if the target is not local), and that the local machine can connect to both the target and the build host. Also, your user must be allowed to copy nix closures between the local machine and the target and host machines. At no point in time are the configuration sources (the nix files) copied anywhere. Instead, nix evaluation always happens locally (with nix-instantiate). The drv-file is then copied and realised remotely (with nix-store). As a convenience, if only --target-host is specified, --build-host is implicitly set to that host too. So if you want to build locally and deploy remotely you have to explicitly set "--build-host localhost". To activate (test, boot or switch) you need to have root access to the target host. You can specify this by "--target-host root@myhost". I have tested the obvious scenarios and they are working. Some of the combinations of --build-host and --target-host and the various actions might not make much sense, and should maybe be forbidden (like setting a remote target host when building a VM), and some combinations might not work at all. --- .../modules/installer/tools/nixos-rebuild.sh | 187 +++++++++++++++--- 1 file changed, 160 insertions(+), 27 deletions(-) diff --git a/nixos/modules/installer/tools/nixos-rebuild.sh b/nixos/modules/installer/tools/nixos-rebuild.sh index 6792690b4c3b..1071460097e8 100644 --- a/nixos/modules/installer/tools/nixos-rebuild.sh +++ b/nixos/modules/installer/tools/nixos-rebuild.sh @@ -19,6 +19,8 @@ rollback= upgrade= repair= profile=/nix/var/nix/profiles/system +buildHost= +targetHost= while [ "$#" -gt 0 ]; do i="$1"; shift 1 @@ -73,6 +75,14 @@ while [ "$#" -gt 0 ]; do fi shift 1 ;; + --build-host|h) + buildHost="$1" + shift 1 + ;; + --target-host|t) + targetHost="$1" + shift 1 + ;; *) echo "$0: unknown option \`$i'" exit 1 @@ -80,6 +90,90 @@ while [ "$#" -gt 0 ]; do esac done + +if [ -z "$buildHost" -a -n "$targetHost" ]; then + buildHost="$targetHost" +fi +if [ "$targetHost" = localhost ]; then + targetHost= +fi +if [ "$buildHost" = localhost ]; then + buildHost= +fi + +buildHostCmd() { + if [ -z "$buildHost" ]; then + "$@" + elif [ -n "$remoteNix" ]; then + ssh $SSHOPTS "$buildHost" PATH="$remoteNix:$PATH" "$@" + else + ssh $SSHOPTS "$buildHost" "$@" + fi +} + +targetHostCmd() { + if [ -z "$targetHost" ]; then + "$@" + else + ssh $SSHOPTS "$targetHost" "$@" + fi +} + +copyToTarget() { + if ! [ "$targetHost" = "$buildHost" ]; then + if [ -z "$targetHost" ]; then + NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --from "$buildHost" "$1" + elif [ -z "$buildHost" ]; then + NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$targetHost" "$1" + else + buildHostCmd nix-copy-closure --to "$targetHost" "$1" + fi + fi +} + +nixBuild() { + if [ -z "$buildHost" ]; then + nix-build "$@" + else + local instArgs=() + local buildArgs=() + + while [ "$#" -gt 0 ]; do + local i="$1"; shift 1 + case "$i" in + -o) + local out="$1"; shift 1 + buildArgs+=("--add-root" "$out" "--indirect") + ;; + -A) + local j="$1"; shift 1 + instArgs+=("$i" "$j") + ;; + -I) + # We don't want this in buildArgs + shift 1 + ;; + "<"*) # nix paths + instArgs+=("$i") + ;; + *) + buildArgs+=("$i") + ;; + esac + done + + local drv="$(nix-instantiate "${instArgs[@]}" "${extraBuildFlags[@]}")" + if [ -a "$drv" ]; then + NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$buildHost" "$drv" + buildHostCmd nix-store -r "$drv" "${buildArgs[@]}" + else + echo "nix-instantiate failed" + exit 1 + fi + fi +} + + if [ -z "$action" ]; then showSyntax; fi # Only run shell scripts from the Nixpkgs tree if the action is @@ -128,7 +222,16 @@ fi tmpDir=$(mktemp -t -d nixos-rebuild.XXXXXX) -trap 'rm -rf "$tmpDir"' EXIT +SSHOPTS="$NIX_SSHOPTS -o ControlMaster=auto -o ControlPath=$tmpDir/ssh-%n -o ControlPersist=60" + +cleanup() { + for ctrl in "$tmpDir"/ssh-*; do + ssh -o ControlPath="$ctrl" -O exit dummyhost 2>/dev/null || true + done + rm -rf "$tmpDir" +} +trap cleanup EXIT + # If the Nix daemon is running, then use it. This allows us to use @@ -150,30 +253,56 @@ if [ -n "$rollback" -o "$action" = dry-build ]; then buildNix= fi +prebuiltNix() { + machine="$1" + if [ "$machine" = x86_64 ]; then + return /nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10 + elif [[ "$machine" =~ i.86 ]]; then + return /nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10 + else + echo "$0: unsupported platform" + exit 1 + fi +} + +remotePATH= + if [ -n "$buildNix" ]; then echo "building Nix..." >&2 - if ! nix-build '' -A config.nix.package -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then - if ! nix-build '' -A nixFallback -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then - if ! nix-build '' -A nix -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then - machine="$(uname -m)" - if [ "$machine" = x86_64 ]; then - nixStorePath=/nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10 - elif [[ "$machine" =~ i.86 ]]; then - nixStorePath=/nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10 - else - echo "$0: unsupported platform" - exit 1 - fi + nixDrv= + if ! nixDrv="$(nix-instantiate '' --add-root $tmpDir/nixdrv --indirect -A config.nix.package "${extraBuildFlags[@]}")"; then + if ! nixDrv="$(nix-instantiate '' --add-root $tmpDir/nixdrv --indirect -A nixFallback "${extraBuildFlags[@]}")"; then + if ! nixDrv="$(nix-instantiate '' --add-root $tmpDir/nixdrv --indirect -A nix "${extraBuildFlags[@]}")"; then + nixStorePath="$(prebuiltNix "$(uname -m)")" if ! nix-store -r $nixStorePath --add-root $tmpDir/nix --indirect \ --option extra-binary-caches https://cache.nixos.org/; then echo "warning: don't know how to get latest Nix" >&2 fi # Older version of nix-store -r don't support --add-root. [ -e $tmpDir/nix ] || ln -sf $nixStorePath $tmpDir/nix + if [ -n "$buildHost" ]; then + remoteNixStorePath="$(prebuiltNix "$(buildHostCmd uname -m)")" + remoteNix="$remoteNixStorePath/bin" + if ! buildHostCmd nix-store -r $remoteNixStorePath \ + --option extra-binary-caches https://cache.nixos.org/ >/dev/null; then + remoteNix= + echo "warning: don't know how to get latest Nix" >&2 + fi + fi fi fi fi - PATH=$tmpDir/nix/bin:$PATH + if [ -a "$nixDrv" ]; then + nix-store -r "$nixDrv" --add-root $tmpDir/nix --indirect >/dev/null + if [ -n "$buildHost" ]; then + nix-copy-closure --to "$buildHost" "$nixDrv" + # The nix build produces multiple outputs, we add them all to the remote path + for p in $(buildHostCmd nix-store -r "$(readlink "$nixDrv")" "${buildArgs[@]}"); do + remoteNix="$remoteNix${remoteNix:+:}$p/bin" + done + fi + fi + PATH="$tmpDir/nix/bin:$PATH" fi @@ -200,31 +329,35 @@ fi if [ -z "$rollback" ]; then echo "building the system configuration..." >&2 if [ "$action" = switch -o "$action" = boot ]; then - nix-env "${extraBuildFlags[@]}" -p "$profile" -f '' --set -A system - pathToConfig="$profile" + pathToConfig="$(nixBuild '' -A system "${extraBuildFlags[@]}")" + copyToTarget "$pathToConfig" + targetHostCmd nix-env -p "$profile" --set "$pathToConfig" elif [ "$action" = test -o "$action" = build -o "$action" = dry-build -o "$action" = dry-activate ]; then - nix-build '' -A system -k "${extraBuildFlags[@]}" > /dev/null - pathToConfig=./result + pathToConfig="$(nixBuild '' -A system -k "${extraBuildFlags[@]}")" elif [ "$action" = build-vm ]; then - nix-build '' -A vm -k "${extraBuildFlags[@]}" > /dev/null - pathToConfig=./result + pathToConfig="$(nixBuild '' -A vm -k "${extraBuildFlags[@]}")" elif [ "$action" = build-vm-with-bootloader ]; then - nix-build '' -A vmWithBootLoader -k "${extraBuildFlags[@]}" > /dev/null - pathToConfig=./result + pathToConfig="$(nixBuild '' -A vmWithBootLoader -k "${extraBuildFlags[@]}")" else showSyntax fi + # Copy build to target host if we haven't already done it + if ! [ "$action" = switch -o "$action" = boot ]; then + copyToTarget "$pathToConfig" + fi else # [ -n "$rollback" ] if [ "$action" = switch -o "$action" = boot ]; then - nix-env --rollback -p "$profile" + targetHostCmd nix-env --rollback -p "$profile" pathToConfig="$profile" elif [ "$action" = test -o "$action" = build ]; then systemNumber=$( - nix-env -p "$profile" --list-generations | + targetHostCmd nix-env -p "$profile" --list-generations | sed -n '/current/ {g; p;}; s/ *\([0-9]*\).*/\1/; h' ) - ln -sT "$profile"-${systemNumber}-link ./result - pathToConfig=./result + pathToConfig="$profile"-${systemNumber}-link + if [ -z "$targetHost" ]; then + ln -sT "$pathToConfig" ./result + fi else showSyntax fi @@ -234,7 +367,7 @@ fi # If we're not just building, then make the new configuration the boot # default and/or activate it now. if [ "$action" = switch -o "$action" = boot -o "$action" = test -o "$action" = dry-activate ]; then - if ! $pathToConfig/bin/switch-to-configuration "$action"; then + if ! targetHostCmd $pathToConfig/bin/switch-to-configuration "$action"; then echo "warning: error(s) occurred while switching to the new configuration" >&2 exit 1 fi