diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 3bb65c6b295a..aa7806a7a9e4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -159,6 +159,7 @@ ./services/audio/ympd.nix ./services/backup/almir.nix ./services/backup/bacula.nix + ./services/backup/borgbackup.nix ./services/backup/crashplan.nix ./services/backup/crashplan-small-business.nix ./services/backup/mysql-backup.nix diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix new file mode 100644 index 000000000000..1b730e0c2b76 --- /dev/null +++ b/nixos/modules/services/backup/borgbackup.nix @@ -0,0 +1,580 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + isLocalPath = x: + builtins.substring 0 1 x == "/" # absolute path + || builtins.substring 0 1 x == "." # relative path + || builtins.match "[.*:.*]" == null; # not machine:path + + mkExcludeFile = cfg: + # Write each exclude pattern to a new line + pkgs.writeText "excludefile" (concatStringsSep "\n" cfg.exclude); + + mkKeepArgs = cfg: + # If cfg.prune.keep e.g. has a yearly attribute, + # its content is passed on as --keep-yearly + concatStringsSep " " + (mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep); + + mkBackupScript = cfg: '' + on_exit() + { + exitStatus=$? + # Reset the EXIT handler, or else we're called again on 'exit' below + trap - EXIT + ${cfg.postHook} + exit $exitStatus + } + trap 'on_exit' INT TERM QUIT EXIT + + archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})" + archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}" + ${cfg.preHook} + '' + optionalString cfg.doInit '' + # Run borg init if the repo doesn't exist yet + if ! borg list > /dev/null; then + borg init \ + --encryption ${cfg.encryption.mode} \ + $extraInitArgs + ${cfg.postInit} + fi + '' + '' + borg create \ + --compression ${cfg.compression} \ + --exclude-from ${mkExcludeFile cfg} \ + $extraCreateArgs \ + "::$archiveName$archiveSuffix" \ + ${escapeShellArgs cfg.paths} + '' + optionalString cfg.appendFailedSuffix '' + borg rename "::$archiveName$archiveSuffix" "$archiveName" + '' + '' + ${cfg.postCreate} + '' + optionalString (cfg.prune.keep != { }) '' + borg prune \ + ${mkKeepArgs cfg} \ + --prefix ${escapeShellArg cfg.prune.prefix} \ + $extraPruneArgs + ${cfg.postPrune} + ''; + + mkPassEnv = cfg: with cfg.encryption; + if passCommand != null then + { BORG_PASSCOMMAND = passCommand; } + else if passphrase != null then + { BORG_PASSPHRASE = passphrase; } + else { }; + + mkBackupService = name: cfg: + let + userHome = config.users.users.${cfg.user}.home; + in nameValuePair "borgbackup-job-${name}" { + description = "BorgBackup job ${name}"; + path = with pkgs; [ + borgbackup openssh + ]; + script = mkBackupScript cfg; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Only run when no other process is using CPU or disk + CPUSchedulingPolicy = "idle"; + IOSchedulingClass = "idle"; + ProtectSystem = "strict"; + ReadWritePaths = + [ "${userHome}/.config/borg" "${userHome}/.cache/borg" ] + # Borg needs write access to repo if it is not remote + ++ optional (isLocalPath cfg.repo) cfg.repo; + PrivateTmp = true; + }; + environment = { + BORG_REPO = cfg.repo; + inherit (cfg) extraInitArgs extraCreateArgs extraPruneArgs; + } // (mkPassEnv cfg) // cfg.environment; + inherit (cfg) startAt; + }; + + # Paths listed in ReadWritePaths must exist before service is started + mkActivationScript = name: cfg: + let + install = "install -o ${cfg.user} -g ${cfg.group}"; + in + nameValuePair "borgbackup-job-${name}" (stringAfter [ "users" ] ('' + # Eensure that the home directory already exists + # We can't assert createHome == true because that's not the case for root + cd "${config.users.users.${cfg.user}.home}" + ${install} -d .config/borg + ${install} -d .cache/borg + '' + optionalString (isLocalPath cfg.repo) '' + ${install} -d ${escapeShellArg cfg.repo} + '')); + + mkPassAssertion = name: cfg: { + assertion = with cfg.encryption; + mode != "none" -> passCommand != null || passphrase != null; + message = + "passCommand or passphrase has to be specified because" + + '' borgbackup.jobs.${name}.encryption != "none"''; + }; + + mkRepoService = name: cfg: + nameValuePair "borgbackup-repo-${name}" { + description = "Create BorgBackup repository ${name} directory"; + script = '' + mkdir -p ${escapeShellArg cfg.path} + chown ${cfg.user}:${cfg.group} ${escapeShellArg cfg.path} + ''; + serviceConfig = { + # The service's only task is to ensure that the specified path exists + Type = "oneshot"; + }; + wantedBy = [ "multi-user.target" ]; + }; + + mkAuthorizedKey = cfg: appendOnly: key: + let + # Because of the following line, clients do not need to specify an absolute repo path + cdCommand = "cd ${escapeShellArg cfg.path}"; + restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} ."; + appendOnlyArg = optionalString appendOnly "--append-only"; + quotaArg = optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}"; + serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}"; + in + ''command="${cdCommand} && ${serveCommand}",restrict ${key}''; + + mkUsersConfig = name: cfg: { + users.${cfg.user} = { + openssh.authorizedKeys.keys = + (map (mkAuthorizedKey cfg false) cfg.authorizedKeys + ++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly); + useDefaultShell = true; + }; + groups.${cfg.group} = { }; + }; + + mkKeysAssertion = name: cfg: { + assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ]; + message = + "borgbackup.repos.${name} does not make sense" + + " without at least one public key"; + }; + +in { + meta.maintainers = with maintainers; [ dotlambda ]; + + ###### interface + + options.services.borgbackup.jobs = mkOption { + description = "Deduplicating backups using BorgBackup."; + default = { }; + example = literalExample '' + { + rootBackup = { + paths = "/"; + exclude = [ "/nix" ]; + repo = "/path/to/local/repo"; + encryption = { + mode = "repokey"; + passphrase = "secret"; + }; + compression = "auto,lzma"; + startAt = "weekly"; + }; + } + ''; + type = types.attrsOf (types.submodule (let globalConfig = config; in + { name, config, ... }: { + options = { + + paths = mkOption { + type = with types; either path (nonEmptyListOf path); + description = "Path(s) to back up."; + example = "/home/user"; + apply = x: if isList x then x else [ x ]; + }; + + repo = mkOption { + type = types.str; + description = "Remote or local repository to back up to."; + example = "user@machine:/path/to/repo"; + }; + + archiveBaseName = mkOption { + type = types.strMatching "[^/{}]+"; + default = "${globalConfig.networking.hostName}-${name}"; + defaultText = "\${config.networking.hostName}-"; + description = '' + How to name the created archives. A timestamp, whose format is + determined by , will be appended. The full + name can be modified at runtime ($archiveName). + Placeholders like {hostname} must not be used. + ''; + }; + + dateFormat = mkOption { + type = types.str; + description = '' + Arguments passed to date + to create a timestamp suffix for the archive name. + ''; + default = "+%Y-%m-%dT%H:%M:%S"; + example = "-u +%s"; + }; + + startAt = mkOption { + type = with types; either str (listOf str); + default = "daily"; + description = '' + When or how often the backup should run. + Must be in the format described in + systemd.time + 7. + If you do not want the backup to start + automatically, use [ ]. + ''; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg is run as. + User or group need read permission + for the specified . + ''; + default = "root"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg is run as. User or group needs read permission + for the specified . + ''; + default = "root"; + }; + + encryption.mode = mkOption { + type = types.enum [ + "repokey" "keyfile" + "repokey-blake2" "keyfile-blake2" + "authenticated" "authenticated-blake2" + "none" + ]; + description = '' + Encryption mode to use. Setting a mode + other than "none" requires + you to specify a + or a . + ''; + }; + + encryption.passCommand = mkOption { + type = with types; nullOr str; + description = '' + A command which prints the passphrase to stdout. + Mutually exclusive with . + ''; + default = null; + example = "cat /path/to/passphrase_file"; + }; + + encryption.passphrase = mkOption { + type = with types; nullOr str; + description = '' + The passphrase the backups are encrypted with. + Mutually exclusive with . + If you do not want the passphrase to be stored in the + world-readable Nix store, use . + ''; + default = null; + }; + + compression = mkOption { + # "auto" is optional, + # compression mode must be given, + # compression level is optional + type = types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?"; + description = '' + Compression method to use. Refer to + borg help compression + for all available options. + ''; + default = "lz4"; + example = "auto,lzma"; + }; + + exclude = mkOption { + type = with types; listOf str; + description = '' + Exclude paths matching any of the given patterns. See + borg help patterns for pattern syntax. + ''; + default = [ ]; + example = [ + "/home/*/.cache" + "/nix" + ]; + }; + + doInit = mkOption { + type = types.bool; + description = '' + Run borg init if the + specified does not exist. + You should set this to false + if the repository is located on an external drive + that might not always be mounted. + ''; + default = true; + }; + + appendFailedSuffix = mkOption { + type = types.bool; + description = '' + Append a .failed suffix + to the archive name, which is only removed if + borg create has a zero exit status. + ''; + default = true; + }; + + prune.keep = mkOption { + # Specifying e.g. `prune.keep.yearly = -1` + # means there is no limit of yearly archives to keep + # The regex is for use with e.g. --keep-within 1y + type = with types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]")); + description = '' + Prune a repository by deleting all archives not matching any of the + specified retention options. See borg help prune + for the available options. + ''; + default = { }; + example = literalExample '' + { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = -1; # Keep at least one archive for each month + } + ''; + }; + + prune.prefix = mkOption { + type = types.str; + description = '' + Only consider archive names starting with this prefix for pruning. + By default, only archives created by this job are considered. + Use "" to consider all archives. + ''; + default = config.archiveBaseName; + defaultText = "\${archiveBaseName}"; + }; + + environment = mkOption { + type = with types; attrsOf str; + description = '' + Environment variables passed to the backup script. + You can for example specify which SSH key to use. + ''; + default = { }; + example = { BORG_RSH = "ssh -i /path/to/key"; }; + }; + + preHook = mkOption { + type = types.lines; + description = '' + Shell commands to run before the backup. + This can for example be used to mount file systems. + ''; + default = ""; + example = '' + # To add excluded paths at runtime + extraCreateArgs="$extraCreateArgs --exclude /some/path" + ''; + }; + + postInit = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg init. + ''; + default = ""; + }; + + postCreate = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg create. The name + of the created archive is stored in $archiveName. + ''; + default = ""; + }; + + postPrune = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg prune. + ''; + default = ""; + }; + + postHook = mkOption { + type = types.lines; + description = '' + Shell commands to run just before exit. They are executed + even if a previous command exits with a non-zero exit code. + The latter is available as $exitStatus. + ''; + default = ""; + }; + + extraInitArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg init. + Can also be set at runtime using $extraInitArgs. + ''; + default = ""; + example = "--append-only"; + }; + + extraCreateArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg create. + Can also be set at runtime using $extraCreateArgs. + ''; + default = ""; + example = "--stats --checkpoint-interval 600"; + }; + + extraPruneArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg prune. + Can also be set at runtime using $extraPruneArgs. + ''; + default = ""; + example = "--save-space"; + }; + + }; + } + )); + }; + + options.services.borgbackup.repos = mkOption { + description = '' + Serve BorgBackup repositories to given public SSH keys, + restricting their access to the repository only. + Also, clients do not need to specify the absolute path when accessing the repository, + i.e. user@machine:. is enough. (Note colon and dot.) + ''; + default = { }; + type = types.attrsOf (types.submodule ( + { name, config, ... }: { + options = { + + path = mkOption { + type = types.path; + description = '' + Where to store the backups. Note that the directory + is created automatically, with correct permissions. + ''; + default = "/var/lib/borgbackup"; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + authorizedKeys = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that are given full write access to this repository. + You should use a different SSH key for each repository you write to, because + the specified keys are restricted to running borg serve + and can only access this single repository. + ''; + default = [ ]; + }; + + authorizedKeysAppendOnly = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that can only be used to append new data (archives) to the repository. + Note that archives can still be marked as deleted and are subsequently removed from disk + upon accessing the repo with full write access, e.g. when pruning. + ''; + default = [ ]; + }; + + allowSubRepos = mkOption { + type = types.bool; + description = '' + Allow clients to create repositories in subdirectories of the + specified . These can be accessed using + user@machine:path/to/subrepo. Note that a + applies to repositories independently. + Therefore, if this is enabled, clients can create multiple + repositories and upload an arbitrary amount of data. + ''; + default = false; + }; + + quota = mkOption { + # See the definition of parse_file_size() in src/borg/helpers/parseformat.py + type = with types; nullOr (strMatching "[[:digit:].]+[KMGTP]?"); + description = '' + Storage quota for the repository. This quota is ensured for all + sub-repositories if is enabled + but not for the overall storage space used. + ''; + default = null; + example = "100G"; + }; + + }; + } + )); + }; + + ###### implementation + + config = mkIf (with config.services.borgbackup; jobs != { } || repos != { }) + (with config.services.borgbackup; { + assertions = + mapAttrsToList mkPassAssertion jobs + ++ mapAttrsToList mkKeysAssertion repos; + + system.activationScripts = mapAttrs' mkActivationScript jobs; + + systemd.services = + # A job named "foo" is mapped to systemd.services.borgbackup-job-foo + mapAttrs' mkBackupService jobs + # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo + // mapAttrs' mkRepoService repos; + + users = mkMerge (mapAttrsToList mkUsersConfig repos); + + environment.systemPackages = with pkgs; [ borgbackup ]; + }); +} diff --git a/nixos/tests/borgbackup.nix b/nixos/tests/borgbackup.nix index 123b02be7251..36731773de27 100644 --- a/nixos/tests/borgbackup.nix +++ b/nixos/tests/borgbackup.nix @@ -1,21 +1,162 @@ -import ./make-test.nix ({ pkgs, ...}: { +import ./make-test.nix ({ pkgs, ... }: + +let + passphrase = "supersecret"; + dataDir = "/ran:dom/data"; + excludeFile = "not_this_file"; + keepFile = "important_file"; + keepFileData = "important_data"; + localRepo = "/root/back:up"; + archiveName = "my_archive"; + remoteRepo = "borg@server:."; # No need to specify path + privateKey = pkgs.writeText "id_ed25519" '' + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrwAAAJB+cF5HfnBe + RwAAAAtzc2gtZWQyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrw + AAAEBN75NsJZSpt63faCuaD75Unko0JjlSDxMhYHAPJk2/xXHxQHThDpD9/AMWNqQer3Tg + 9gXMb2lTZMn0pelo8xyvAAAADXJzY2h1ZXR6QGt1cnQ= + -----END OPENSSH PRIVATE KEY----- + ''; + publicKey = '' + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHxQHThDpD9/AMWNqQer3Tg9gXMb2lTZMn0pelo8xyv root@client + ''; + privateKeyAppendOnly = pkgs.writeText "id_ed25519" '' + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACBacZuz1ELGQdhI7PF6dGFafCDlvh8pSEc4cHjkW0QjLwAAAJC9YTxxvWE8 + cQAAAAtzc2gtZWQyNTUxOQAAACBacZuz1ELGQdhI7PF6dGFafCDlvh8pSEc4cHjkW0QjLw + AAAEAAhV7wTl5dL/lz+PF/d4PnZXuG1Id6L/mFEiGT1tZsuFpxm7PUQsZB2Ejs8Xp0YVp8 + IOW+HylIRzhweORbRCMvAAAADXJzY2h1ZXR6QGt1cnQ= + -----END OPENSSH PRIVATE KEY----- + ''; + publicKeyAppendOnly = '' + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFpxm7PUQsZB2Ejs8Xp0YVp8IOW+HylIRzhweORbRCMv root@client + ''; + +in { name = "borgbackup"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ mic92 ]; + meta = with pkgs.stdenv.lib; { + maintainers = with maintainers; [ dotlambda ]; }; nodes = { - machine = { config, pkgs, ... }: { - environment.systemPackages = [ pkgs.borgbackup ]; + client = { config, pkgs, ... }: { + services.borgbackup.jobs = { + + local = rec { + paths = dataDir; + repo = localRepo; + preHook = '' + # Don't append a timestamp + archiveName="${archiveName}" + ''; + encryption = { + mode = "repokey"; + inherit passphrase; + }; + compression = "auto,zlib,9"; + prune.keep = { + within = "1y"; + yearly = 5; + }; + exclude = [ "*/${excludeFile}" ]; + postHook = "echo post"; + startAt = [ ]; # Do not run automatically + }; + + remote = { + paths = dataDir; + repo = remoteRepo; + encryption.mode = "none"; + startAt = [ ]; + environment.BORG_RSH = "ssh -oStrictHostKeyChecking=no -i /root/id_ed25519"; + }; + + remoteAppendOnly = { + paths = dataDir; + repo = remoteRepo; + encryption.mode = "none"; + startAt = [ ]; + environment.BORG_RSH = "ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly"; + }; + + }; + }; + + server = { config, pkgs, ... }: { + services.openssh = { + enable = true; + passwordAuthentication = false; + challengeResponseAuthentication = false; + }; + + services.borgbackup.repos.repo1 = { + authorizedKeys = [ publicKey ]; + path = "/data/borgbackup"; + }; + + # Second repo to make sure the authorizedKeys options are merged correctly + services.borgbackup.repos.repo2 = { + authorizedKeysAppendOnly = [ publicKeyAppendOnly ]; + path = "/data/borgbackup"; + quota = ".5G"; + }; }; }; testScript = '' - my $borg = "BORG_PASSPHRASE=supersecret borg"; - $machine->succeed("$borg init --encryption=repokey /tmp/backup"); - $machine->succeed("mkdir /tmp/data/ && echo 'data' >/tmp/data/file"); - $machine->succeed("$borg create --stats /tmp/backup::test /tmp/data"); - $machine->succeed("$borg extract /tmp/backup::test"); - $machine->succeed('c=$(cat data/file) && echo "c = $c" >&2 && [[ "$c" == "data" ]]'); + startAll; + + $client->fail('test -d "${remoteRepo}"'); + + $client->succeed("cp ${privateKey} /root/id_ed25519"); + $client->succeed("chmod 0600 /root/id_ed25519"); + $client->succeed("cp ${privateKeyAppendOnly} /root/id_ed25519.appendOnly"); + $client->succeed("chmod 0600 /root/id_ed25519.appendOnly"); + + $client->succeed("mkdir -p ${dataDir}"); + $client->succeed("touch ${dataDir}/${excludeFile}"); + $client->succeed("echo '${keepFileData}' > ${dataDir}/${keepFile}"); + + subtest "local", sub { + my $borg = "BORG_PASSPHRASE='${passphrase}' borg"; + $client->systemctl("start --wait borgbackup-job-local"); + $client->fail("systemctl is-failed borgbackup-job-local"); + # Make sure exactly one archive has been created + $client->succeed("c=\$($borg list '${localRepo}' | wc -l) && [[ \$c == '1' ]]"); + # Make sure excludeFile has been excluded + $client->fail("$borg list '${localRepo}::${archiveName}' | grep -qF '${excludeFile}'"); + # Make sure keepFile has the correct content + $client->succeed("$borg extract '${localRepo}::${archiveName}'"); + $client->succeed('c=$(cat ${dataDir}/${keepFile}) && [[ "$c" == "${keepFileData}" ]]'); + }; + + subtest "remote", sub { + my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519' borg"; + $server->waitForUnit("sshd.service"); + $client->waitForUnit("network.target"); + $client->systemctl("start --wait borgbackup-job-remote"); + $client->fail("systemctl is-failed borgbackup-job-remote"); + + # Make sure we can't access repos other than the specified one + $client->fail("$borg list borg\@server:wrong"); + + #TODO: Make sure that data is actually deleted + }; + + subtest "remoteAppendOnly", sub { + my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly' borg"; + $server->waitForUnit("sshd.service"); + $client->waitForUnit("network.target"); + $client->systemctl("start --wait borgbackup-job-remoteAppendOnly"); + $client->fail("systemctl is-failed borgbackup-job-remoteAppendOnly"); + + # Make sure we can't access repos other than the specified one + $client->fail("$borg list borg\@server:wrong"); + + #TODO: Make sure that data is not actually deleted + }; + ''; })