From 3e0c9ed79347f119ba494ff858ee81ee078ac9cc Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 8 Jul 2022 14:03:55 +0200 Subject: [PATCH] add borg backup --- backup.nix | 121 ++++++++++++++++++ configuration.nix | 7 +- deployment/README.md | 4 + deployment/deploy-backup.sh | 59 +++++++++ deployment/deploy.sh | 36 +++++- deployment/flake.nix | 16 +++ krops/deploy.nix | 1 + maintenance/maintenance.sh | 29 +++++ matrix.nix | 6 - .../nixbitcoin.org/backup-encryption-env.gpg | Bin 1694 -> 0 bytes 10 files changed, 264 insertions(+), 15 deletions(-) create mode 100644 backup.nix create mode 100644 deployment/deploy-backup.sh delete mode 100644 secrets/nixbitcoin.org/backup-encryption-env.gpg diff --git a/backup.nix b/backup.nix new file mode 100644 index 0000000..20d40cb --- /dev/null +++ b/backup.nix @@ -0,0 +1,121 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + secretsDir = config.nix-bitcoin.secretsDir; +in +{ + services.zfs.autoSnapshot.enable = true; + + # Only use daily, weekly, monthly ZFS snapshots + systemd.timers = { + zfs-snapshot-frequent.enable = false; + zfs-snapshot-hourly.enable = false; + # The daily snapshot is run by borgbackup-job-main.service + zfs-snapshot-daily.enable = false; + }; + + systemd.services.borgbackup-job-main = rec { + requires = [ "zfs-snapshot-daily.service" ]; + after = requires; + }; + + programs.ssh.knownHosts."freak.seedhost.eu".publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD2TmdE89ZD4XshcIcXZPLFC/nDxZdAr9yrH2/2OCNKEo/Ex60y8TQjp93isjdDj7Grf/GpW60OONfXTFe0r5iM="; + + services.borgbackup.jobs = { + main = { + startAt = "daily"; + + preHook = '' + latest_daily_snap=$( + shopt -s nullglob + printf '%s\n' /.zfs/snapshot/*daily* | tail -1 + ) + if [[ ! $latest_daily_snap ]]; then + echo "Error: No daily snapshot found" + exit 1 + fi + echo "Using $latest_daily_snap" + cd $latest_daily_snap + ''; + + paths = [ "var/lib" ]; + + exclude = [ + "var/lib/bitcoind/blocks" + "var/lib/bitcoind/chainstate" + "var/lib/bitcoind/indexes" + "var/lib/liquidd/*/blocks" + "var/lib/liquidd/*/chainstate" + "var/lib/liquidd/*/indexes" + "var/lib/electrs" + "var/lib/fulcrum" + "var/lib/nbxplorer" + "var/lib/duplicity" + "var/lib/onion-addresses" + + "var/lib/i2pd" + "var/lib/redis" + "var/lib/udisks2" + "var/lib/usbguard" + + "var/lib/acme" + "var/lib/dovecot" + "var/lib/dhparams" # from dovecot + "var/lib/postfix" + "var/lib/rspamd" + + "var/lib/machines" + "var/lib/private" + "var/lib/systemd" + ]; + + repo = "nixbitcoin@freak.seedhost.eu:borg-backup"; + doInit = false; + encryption = { + mode = "repokey"; + passCommand = "cat ${secretsDir}/backup-encryption-password"; + }; + environment = { + BORG_RSH = "ssh -i ${secretsDir}/ssh-key-seedhost"; + # TODO-EXTERNAL: Use this definition when the borg job wrapper script + # has been fixed in the borgbackup.nix NixOS module + # BORG_REMOTE_PATH = "$HOME/.local/bin/borg"; + BORG_REMOTE_PATH = "/home34/nixbitcoin/.local/bin/borg"; + }; + compression = "zstd"; + extraCreateArgs = "--stats"; # Print stats after backup + prune.keep = { + within = "1d"; # Keep all archives from the last day + daily = 4; + weekly = 2; + monthly = 2; + }; + # Compact (free repo storage space) every 7 days + postPrune = '' + if (( (($(date +%s) / 86400) % 7) == 0 )); then + borg compact + fi + ''; + }; + }; + + nix-bitcoin.secrets = { + backup-encryption-password.user = "root"; + ssh-key-seedhost = { + user = "root"; + permissions = "400"; + }; + }; + nix-bitcoin.generateSecretsCmds.backups = '' + makePasswordSecret backup-encryption-password + # Generate a dummy file so that setup-secrets doesn't fail + touch ssh-key-seedhost + ''; + + # Use borg 1.2.1 (the latest 1.2.* release) + # TODO-EXTERNAL: Remove this when 1.2.1 has landed in nixpkgs stable + nixpkgs.overlays = [ + (_: _: { inherit (config.nix-bitcoin.pkgs.pkgsUnstable) borgbackup; }) + ]; +} diff --git a/configuration.nix b/configuration.nix index 9c12991..e33b9c8 100644 --- a/configuration.nix +++ b/configuration.nix @@ -6,6 +6,7 @@ ./base.nix ./website ./matrix.nix + ./backup.nix ]; services.zfs.autoScrub = { @@ -67,12 +68,6 @@ donate.btcpayserverAppId = "3NKhG5wANegkfmXJ5x4ZNuSAB1z5"; }; - services.backups = { - enable = true; - destination = "sftp://nixbitcoin@freak.seedhost.eu"; - }; - programs.ssh.knownHosts."freak.seedhost.eu".publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD2TmdE89ZD4XshcIcXZPLFC/nDxZdAr9yrH2/2OCNKEo/Ex60y8TQjp93isjdDj7Grf/GpW60OONfXTFe0r5iM="; - environment.shellAliases = { sudo = "doas"; }; diff --git a/deployment/README.md b/deployment/README.md index 1548847..6816f9a 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -13,3 +13,7 @@ These steps are implemented in: [`./deploy.sh`](./deploy.sh) ### Test deployment via a local VM See [`./vm-deployment.sh`](./vm-deployment.sh) + +## Backups + +See [`./deploy-backup.sh`](./deploy-backup.sh) diff --git a/deployment/deploy-backup.sh b/deployment/deploy-backup.sh new file mode 100644 index 0000000..3e92756 --- /dev/null +++ b/deployment/deploy-backup.sh @@ -0,0 +1,59 @@ +## +## Warning: This is a very rough sketch +## + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# 1. Deploy borg binary on seedhost + +TMPDIR=$(mktemp -d) +export GNUPGHOME=$TMPDIR + +# fetch dev key (tw@waldmann-edv.de) +gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 9F88FB52FAF7B393 + +# Download `linuxold` release which is compatible with the glibc version on seedhost +curl -L https://github.com/borgbackup/borg/releases/download/1.2.1/borg-linuxold64 -O +curl -L https://github.com/borgbackup/borg/releases/download/1.2.1/borg-linuxold64.asc -O +gpg --verify borg-linuxold64.asc + +rm -rf $TMPDIR + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# 2. Init borg backup repo on seedhost + +# Run the following commands in a nix shell +nix shell nixpkgs/29399e5ad1660668b61247c99894fc2fb97b4e74#borgbackup + +# Ensure borg version is >= 1.2.1 +borg --version + +if [[ ! $XDG_RUNTIME_DIR ]]; then + echo 'Error: Missing env var XDG_RUNTIME_DIR' + return 1 +fi +export secrets=$XDG_RUNTIME_DIR/borg-secrets +mkdir -p $secrets +install -m 600 <(gpg --decrypt ../secrets/nixbitcoin.org/ssh-key-seedhost.gpg) $secrets/ssh-key-seedhost +install -m 600 <(gpg --decrypt ../secrets/nixbitcoin.org/backup-encryption-password.gpg) $secrets/backup-encryption-password + +export BORG_REPO=nixbitcoin@freak.seedhost.eu:borg-backup +export BORG_RSH="ssh -i $secrets/ssh-key-seedhost" +export BORG_REMOTE_PATH='$HOME/.local/bin/borg' +export BORG_PASSCOMMAND="cat $secrets/backup-encryption-password" +borg init --encryption=repokey --storage-quota=400G + +debugCmds() { + borg info + + ssh freak.seedhost.eu 'ls -alt' + ssh freak.seedhost.eu 'ls -al borg-backup' + ssh freak.seedhost.eu 'ls -al borg-backup/data' + # Delete repo + ssh freak.seedhost.eu 'rm -rf borg-backup' + + # This can be run on nixbitoin.org to manage the backup + borg-job-main ... + borg-job-main list +} + +rm -rf $secrets diff --git a/deployment/deploy.sh b/deployment/deploy.sh index 3692244..66c17b5 100644 --- a/deployment/deploy.sh +++ b/deployment/deploy.sh @@ -136,10 +136,40 @@ rm -rf /tmp/deploy-nixbitcoinorg #――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # Optional: Restore backups -# Enter server -ssh nixbitcoin.org +## Variant 1: Restore /var/lib from another system +ssh nixbitcoin.org 'rsync -ahz --info=progress2 backup-server:/temp/var/lib/ /var/lib --exclude=/systemd' -rsync -ahz --info=progress2 backup-server:/temp/var/lib/ /var/lib --exclude=/systemd +## Variant 2: Restore /var/lib from backup +# TODO: Streamline this deployment step when switching to flakes for the main system deployment + +# Copy backup-related secrets +gpg --decrypt ../secrets/nixbitcoin.org/backup-encryption-password.gpg 2>/dev/null | \ + ssh nixbitcoin.org 'install -D -m 600 <(cat) /var/src/secrets/backup-encryption-password' +gpg --decrypt ../secrets/nixbitcoin.org/ssh-key-seedhost.gpg 2>/dev/null | \ + ssh nixbitcoin.org 'install -D -m 600 <(cat) /var/src/secrets/ssh-key-seedhost' + +deployBaseSystemWithBackups() {( + set -euxo pipefail + nix build .#packages.x86_64-linux.baseSystemWithBackups --out-link /tmp/deploy-nixbitcoinorg/system + nix copy --to ssh://nixbitcoin.org /tmp/deploy-nixbitcoinorg/system + ssh -n nixbitcoin.org "$(realpath /tmp/deploy-nixbitcoinorg/system)/bin/switch-to-configuration switch" +)} +deployBaseSystemWithBackups + +# List backups +ssh nixbitcoin.org borg-job-main list +# Show last backup +ssh nixbitcoin.org 'borg-job-main info ::$(borg-job-main list --short | tail -1)' + +# Restore last backup +# As of 2022-07-01, the latest backup had a compressed size of 19.02 GB (54.79 GB uncompressed). +# Restoring from seedhost took 6m33.127s. +ssh nixbitcoin.org bash -s <<'EOF' +tmux new -d -s restore-backup 'cd / && time borg-job-main extract --progress ::$(borg-job-main list --short | tail -1); sleep infinity' +EOF + +# Check progress. Close terminal or press 'Ctrl+b d' to detach +ssh -t nixbitcoin.org 'tmux attach' #――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # Step 3: Install main system diff --git a/deployment/flake.nix b/deployment/flake.nix index fcc1e53..e4b019d 100644 --- a/deployment/flake.nix +++ b/deployment/flake.nix @@ -39,6 +39,22 @@ inherit system; modules = [ ../base.nix ]; }).config.system.build.toplevel; + + baseSystemWithBackups = (nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + ../base.nix + ../backup.nix + nix-bitcoin.nixosModule + ({lib, ... } :{ + services.borgbackup.jobs.main.startAt = lib.mkForce []; + nix-bitcoin = { + secretsDir = "/var/src/secrets"; + setupSecrets = true; + }; + }) + ]; + }).config.system.build.toplevel; }; }) // { nixosConfigurations = { diff --git a/krops/deploy.nix b/krops/deploy.nix index b372c6d..c8adb64 100644 --- a/krops/deploy.nix +++ b/krops/deploy.nix @@ -8,6 +8,7 @@ let website.file = { path = toString ../website; }; + "backup.nix".file = toString ../backup.nix; "matrix.nix".file = toString ../matrix.nix; "mail.nix".file = toString ../mail.nix; }; diff --git a/maintenance/maintenance.sh b/maintenance/maintenance.sh index 487a71b..d350edb 100644 --- a/maintenance/maintenance.sh +++ b/maintenance/maintenance.sh @@ -60,6 +60,35 @@ xdg-open http://localhost:10000 gpg --decrypt ../secrets/client-side/btcpayserver-credentials.gpg 2>/dev/null xdg-open http://localhost:10001/btcpayserver +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Backups +# https://borgbackup.readthedocs.io/en/stable +# See also: ../deployment/deploy.sh (Restore backups) + +journalctl -u borgbackup-job-main -n 50 + +# show repo stats +borg-job-main info +# list backups +borg-job-main list + +## inspect backups +# show stats of last backup +borg-job-main info ::$(borg-job-main list --short | tail -1) +# show first 10 files of last backup +borg-job-main list ::$(borg-job-main list --short | tail -1) | head -10 +# show specific paths +borg-job-main list ::$(borg-job-main list --short | tail -1) var/lib/clightning +# diff contents of penultimate backup with the last backup +backups=$(borg-job-main list --short); borg-job-main diff ::$(<<<"$backups" tail -2 | head -1) $(<<<"$backups" tail -1) + +## restore files +# restore a path from last backup (dry-run) +borg-job-main extract --dry-run --progress --list ::$(borg-job-main list --short | tail -1) var/lib/clightning + +# show specific file content from last backup +borg-job-main extract --stdout ::$(borg-job-main list --short | tail -1) var/lib/clightning/config + #――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # Update postgresql database schema # Required when the postgresql version is updated when changing `system.stateVersion` diff --git a/matrix.nix b/matrix.nix index ecc3085..b86c1aa 100644 --- a/matrix.nix +++ b/matrix.nix @@ -231,12 +231,6 @@ in { }; }; - # Backups - services.backups = { - extraFiles = [ dataDir ]; - postgresqlDatabases = [ "matrix-synapse" ]; - }; - nix-bitcoin.secrets.matrix-smtp-password.user = "matrix-synapse"; # Used by dovecot2 (via the mailserver module) nix-bitcoin.secrets.matrix-smtp-password-hashed.user = "root"; diff --git a/secrets/nixbitcoin.org/backup-encryption-env.gpg b/secrets/nixbitcoin.org/backup-encryption-env.gpg deleted file mode 100644 index 0cdd3734c7f1146312ca07e0a738059adb59d951..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1694 zcmV;P24VSy0t^EH00000000015C2t%vAM3A*Ga&j5RA;8D0!Gx=uVi}s9zCBPxSSl z)?IR3vrCDRw85**5S-ivRNnxqf?t#AeQZ6}xauU<5=xn3BDd9DmHV0Af5A5mFI=Xu zO!b_A`50fEO&Cgy7Y$e|-a-8?f(tG6|<3pj~EI3YX}xK*c}rD8%wnNqpg6*F{W3&ln{o zWU`tr^+qy}YcF%@=Y00ws1UWs2DmR(__x(kI^i&Zm&kFNQQHTdkRVu6B8dut3&|~zoob$Mb}=AdlcNiEkU#eH5fL%&jPh%uSuVzC%BmSYx6G6OCYHN5DjI<@CK*BP{Ov{m<1Az63%vvp%*i7gxQz=k@4l_I(elT6^* z7u0}XcA!FA*vBsvNeB1*t7OnGA=d@)m?rA@!2(sooW)RaFAqVU=)&6lIN>~9s+nti zAf9UXx^W%BXCG6Ea&nP+(UZQwlVMZ{cnlMa)#03#9E;yAT7cg0_zWf~N!H%+Gg{(c zKn}o(x4oKPwASU`1uB+tFXlE&z9$em1wE`Vqg%}l=Mb3;$7Nw~ZP-bvV%hN8+J^1I z#mHZ$AH`9=X08C8mpvwqPFHJ}Z=ajE07GQwc*B}VnvqXX72f++%-}UlL2@bWJljn; zsBl3xM4gFe0{{R300000009sD3V!TQ-KBSkn7RKzSfysP zH8woM++KCkXP2clyX!+${wOWNOO7&_BP{u!XkItfW6uq?>+W~@6I++^n9AW%)>fkK z;x^k+Ma{@2f@f6NYn)FiFBlT>XDtgdxEz&z7w`D^^1`Tj&V~ZW6uTeY$+&^A>eKi> z5rtC-0?|Dn3~ap|-#?|>=_4kmsb6C8IZ)ba$qXQ+Pz65z_ODWs_}nUz@-cdqWh#8deY-GXWVV7NWFq8WBCT{3YK2?q^F<93OZ0Qv1W`% z+Kg_FAoT_}yPR%7RI;l?exSr6RxXzpLogvjNt`>eSTjVNf+T|W5TWiRHHN0y$@I;{ zN$2$AN&q?EYmlHgX(5@3b5x9iDl}h`G!i8%t3?g&KPbk1c2>2K>*AJ4v7;)3vL@H1 zSrf0pY&3`STBnTC7Yq);>sVeZ!F>RgEEdl8(r*F2xex{hKn^(AW`pICr{co)dndEb zpj%pVY+4MdO~UKkVOcC`gfSUby>q0alhDrp_pg7!DiQ%SWfTrUC*T> o_PmfG1sHCJ?n7==-}oGly-v02%NlB+3Ij-$YS{D)gL5K6!s3QQb^rhX