add borg backup

backup.nix

@@ -0,0 +1,121 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  secretsDir = config.nix-bitcoin.secretsDir;
+in
+{
+  services.zfs.autoSnapshot.enable = true;
+
+  # Only use daily, weekly, monthly ZFS snapshots
+  systemd.timers = {
+    zfs-snapshot-frequent.enable = false;
+    zfs-snapshot-hourly.enable = false;
+    # The daily snapshot is run by borgbackup-job-main.service
+    zfs-snapshot-daily.enable = false;
+  };
+
+  systemd.services.borgbackup-job-main = rec {
+    requires = [ "zfs-snapshot-daily.service" ];
+    after = requires;
+  };
+
+  programs.ssh.knownHosts."freak.seedhost.eu".publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD2TmdE89ZD4XshcIcXZPLFC/nDxZdAr9yrH2/2OCNKEo/Ex60y8TQjp93isjdDj7Grf/GpW60OONfXTFe0r5iM=";
+
+  services.borgbackup.jobs = {
+    main = {
+      startAt = "daily";
+
+      preHook = ''
+        latest_daily_snap=$(
+          shopt -s nullglob
+          printf '%s\n' /.zfs/snapshot/*daily* | tail -1
+        )
+        if [[ ! $latest_daily_snap ]]; then
+          echo "Error: No daily snapshot found"
+          exit 1
+        fi
+        echo "Using $latest_daily_snap"
+        cd $latest_daily_snap
+      '';
+
+      paths = [ "var/lib" ];
+
+      exclude = [
+        "var/lib/bitcoind/blocks"
+        "var/lib/bitcoind/chainstate"
+        "var/lib/bitcoind/indexes"
+        "var/lib/liquidd/*/blocks"
+        "var/lib/liquidd/*/chainstate"
+        "var/lib/liquidd/*/indexes"
+        "var/lib/electrs"
+        "var/lib/fulcrum"
+        "var/lib/nbxplorer"
+        "var/lib/duplicity"
+        "var/lib/onion-addresses"
+
+        "var/lib/i2pd"
+        "var/lib/redis"
+        "var/lib/udisks2"
+        "var/lib/usbguard"
+
+        "var/lib/acme"
+        "var/lib/dovecot"
+        "var/lib/dhparams" # from dovecot
+        "var/lib/postfix"
+        "var/lib/rspamd"
+
+        "var/lib/machines"
+        "var/lib/private"
+        "var/lib/systemd"
+      ];
+
+      repo = "nixbitcoin@freak.seedhost.eu:borg-backup";
+      doInit = false;
+      encryption = {
+        mode = "repokey";
+        passCommand = "cat ${secretsDir}/backup-encryption-password";
+      };
+      environment = {
+        BORG_RSH = "ssh -i ${secretsDir}/ssh-key-seedhost";
+        # TODO-EXTERNAL: Use this definition when the borg job wrapper script
+        # has been fixed in the borgbackup.nix NixOS module
+        # BORG_REMOTE_PATH = "$HOME/.local/bin/borg";
+        BORG_REMOTE_PATH = "/home34/nixbitcoin/.local/bin/borg";
+      };
+      compression = "zstd";
+      extraCreateArgs = "--stats"; # Print stats after backup
+      prune.keep = {
+        within = "1d"; # Keep all archives from the last day
+        daily = 4;
+        weekly = 2;
+        monthly = 2;
+      };
+      # Compact (free repo storage space) every 7 days
+      postPrune = ''
+        if (( (($(date +%s) / 86400) % 7) == 0 )); then
+          borg compact
+        fi
+      '';
+    };
+  };
+
+  nix-bitcoin.secrets = {
+    backup-encryption-password.user = "root";
+    ssh-key-seedhost = {
+      user = "root";
+      permissions = "400";
+    };
+  };
+  nix-bitcoin.generateSecretsCmds.backups = ''
+     makePasswordSecret backup-encryption-password
+     # Generate a dummy file so that setup-secrets doesn't fail
+     touch ssh-key-seedhost
+  '';
+
+  # Use borg 1.2.1 (the latest 1.2.* release)
+  # TODO-EXTERNAL: Remove this when 1.2.1 has landed in nixpkgs stable
+  nixpkgs.overlays = [
+    (_: _: { inherit (config.nix-bitcoin.pkgs.pkgsUnstable) borgbackup; })
+  ];
+}

configuration.nix

@@ -6,6 +6,7 @@
     ./base.nix
     ./website
     ./matrix.nix
+    ./backup.nix
   ];
 
   services.zfs.autoScrub = {
@@ -67,12 +68,6 @@
     donate.btcpayserverAppId = "3NKhG5wANegkfmXJ5x4ZNuSAB1z5";
   };
 
-  services.backups = {
-    enable = true;
-    destination = "sftp://nixbitcoin@freak.seedhost.eu";
-  };
-  programs.ssh.knownHosts."freak.seedhost.eu".publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD2TmdE89ZD4XshcIcXZPLFC/nDxZdAr9yrH2/2OCNKEo/Ex60y8TQjp93isjdDj7Grf/GpW60OONfXTFe0r5iM=";
-
   environment.shellAliases = {
     sudo = "doas";
   };

deployment/README.md

@@ -13,3 +13,7 @@ These steps are implemented in: [`./deploy.sh`](./deploy.sh)
 ### Test deployment via a local VM
 
 See [`./vm-deployment.sh`](./vm-deployment.sh)
+
+## Backups
+
+See [`./deploy-backup.sh`](./deploy-backup.sh)

deployment/deploy-backup.sh

@@ -0,0 +1,59 @@
+##
+## Warning: This is a very rough sketch
+##
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# 1. Deploy borg binary on seedhost
+
+TMPDIR=$(mktemp -d)
+export GNUPGHOME=$TMPDIR
+
+# fetch dev key (tw@waldmann-edv.de)
+gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 9F88FB52FAF7B393
+
+# Download `linuxold` release which is compatible with the glibc version on seedhost
+curl -L https://github.com/borgbackup/borg/releases/download/1.2.1/borg-linuxold64 -O
+curl -L https://github.com/borgbackup/borg/releases/download/1.2.1/borg-linuxold64.asc -O
+gpg --verify borg-linuxold64.asc
+
+rm -rf $TMPDIR
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# 2. Init borg backup repo on seedhost
+
+# Run the following commands in a nix shell
+nix shell nixpkgs/29399e5ad1660668b61247c99894fc2fb97b4e74#borgbackup
+
+# Ensure borg version is >= 1.2.1
+borg --version
+
+if [[ ! $XDG_RUNTIME_DIR ]]; then
+    echo 'Error: Missing env var XDG_RUNTIME_DIR'
+    return 1
+fi
+export secrets=$XDG_RUNTIME_DIR/borg-secrets
+mkdir -p $secrets
+install -m 600 <(gpg --decrypt ../secrets/nixbitcoin.org/ssh-key-seedhost.gpg) $secrets/ssh-key-seedhost
+install -m 600 <(gpg --decrypt ../secrets/nixbitcoin.org/backup-encryption-password.gpg) $secrets/backup-encryption-password
+
+export BORG_REPO=nixbitcoin@freak.seedhost.eu:borg-backup
+export BORG_RSH="ssh -i $secrets/ssh-key-seedhost"
+export BORG_REMOTE_PATH='$HOME/.local/bin/borg'
+export BORG_PASSCOMMAND="cat $secrets/backup-encryption-password"
+borg init --encryption=repokey --storage-quota=400G
+
+debugCmds() {
+    borg info
+
+    ssh freak.seedhost.eu 'ls -alt'
+    ssh freak.seedhost.eu 'ls -al borg-backup'
+    ssh freak.seedhost.eu 'ls -al borg-backup/data'
+    # Delete repo
+    ssh freak.seedhost.eu 'rm -rf borg-backup'
+
+    # This can be run on nixbitoin.org to manage the backup
+    borg-job-main ...
+    borg-job-main list
+}
+
+rm -rf $secrets

deployment/deploy.sh

@@ -136,10 +136,40 @@ rm -rf /tmp/deploy-nixbitcoinorg
 #―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
 # Optional: Restore backups
 
-# Enter server
-ssh nixbitcoin.org
+## Variant 1: Restore /var/lib from another system
+ssh nixbitcoin.org 'rsync -ahz --info=progress2 backup-server:/temp/var/lib/ /var/lib --exclude=/systemd'
 
-rsync -ahz --info=progress2 backup-server:/temp/var/lib/ /var/lib --exclude=/systemd
+## Variant 2: Restore /var/lib from backup
+# TODO: Streamline this deployment step when switching to flakes for the main system deployment
+
+# Copy backup-related secrets
+gpg --decrypt ../secrets/nixbitcoin.org/backup-encryption-password.gpg 2>/dev/null | \
+    ssh nixbitcoin.org 'install -D -m 600 <(cat) /var/src/secrets/backup-encryption-password'
+gpg --decrypt ../secrets/nixbitcoin.org/ssh-key-seedhost.gpg 2>/dev/null | \
+    ssh nixbitcoin.org 'install -D -m 600 <(cat) /var/src/secrets/ssh-key-seedhost'
+
+deployBaseSystemWithBackups() {(
+  set -euxo pipefail
+  nix build .#packages.x86_64-linux.baseSystemWithBackups --out-link /tmp/deploy-nixbitcoinorg/system
+  nix copy --to ssh://nixbitcoin.org /tmp/deploy-nixbitcoinorg/system
+  ssh -n nixbitcoin.org "$(realpath /tmp/deploy-nixbitcoinorg/system)/bin/switch-to-configuration switch"
+)}
+deployBaseSystemWithBackups
+
+# List backups
+ssh nixbitcoin.org borg-job-main list
+# Show last backup
+ssh nixbitcoin.org 'borg-job-main info ::$(borg-job-main list --short | tail -1)'
+
+# Restore last backup
+# As of 2022-07-01, the latest backup had a compressed size of 19.02 GB (54.79 GB uncompressed).
+# Restoring from seedhost took 6m33.127s.
+ssh nixbitcoin.org bash -s <<'EOF'
+tmux new -d -s restore-backup 'cd / && time borg-job-main extract --progress ::$(borg-job-main list --short | tail -1); sleep infinity'
+EOF
+
+# Check progress. Close terminal or press 'Ctrl+b d' to detach
+ssh -t nixbitcoin.org 'tmux attach'
 
 #―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
 # Step 3: Install main system

deployment/flake.nix

@@ -39,6 +39,22 @@
             inherit system;
             modules = [ ../base.nix ];
           }).config.system.build.toplevel;
+
+          baseSystemWithBackups = (nixpkgs.lib.nixosSystem {
+            inherit system;
+            modules = [
+              ../base.nix
+              ../backup.nix
+              nix-bitcoin.nixosModule
+              ({lib, ... } :{
+                services.borgbackup.jobs.main.startAt = lib.mkForce [];
+                nix-bitcoin = {
+                  secretsDir = "/var/src/secrets";
+                  setupSecrets = true;
+                };
+              })
+            ];
+          }).config.system.build.toplevel;
         };
       }) // {
         nixosConfigurations = {

krops/deploy.nix

@@ -8,6 +8,7 @@ let
     website.file = {
       path = toString ../website;
     };
+    "backup.nix".file = toString ../backup.nix;
     "matrix.nix".file = toString ../matrix.nix;
     "mail.nix".file = toString ../mail.nix;
   };

maintenance/maintenance.sh

@@ -60,6 +60,35 @@ xdg-open http://localhost:10000
 gpg --decrypt ../secrets/client-side/btcpayserver-credentials.gpg 2>/dev/null
 xdg-open http://localhost:10001/btcpayserver
 
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Backups
+# https://borgbackup.readthedocs.io/en/stable
+# See also: ../deployment/deploy.sh (Restore backups)
+
+journalctl -u borgbackup-job-main -n 50
+
+# show repo stats
+borg-job-main info
+# list backups
+borg-job-main list
+
+## inspect backups
+# show stats of last backup
+borg-job-main info ::$(borg-job-main list --short | tail -1)
+# show first 10 files of last backup
+borg-job-main list ::$(borg-job-main list --short | tail -1) | head -10
+# show specific paths
+borg-job-main list ::$(borg-job-main list --short | tail -1) var/lib/clightning
+# diff contents of penultimate backup with the last backup
+backups=$(borg-job-main list --short); borg-job-main diff ::$(<<<"$backups" tail -2 | head -1) $(<<<"$backups" tail -1)
+
+## restore files
+# restore a path from last backup (dry-run)
+borg-job-main extract --dry-run --progress --list ::$(borg-job-main list --short | tail -1) var/lib/clightning
+
+# show specific file content from last backup
+borg-job-main extract --stdout ::$(borg-job-main list --short | tail -1) var/lib/clightning/config
+
 #―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
 # Update postgresql database schema
 # Required when the postgresql version is updated when changing `system.stateVersion`

matrix.nix

@@ -231,12 +231,6 @@ in {
     };
   };
 
-  # Backups
-  services.backups = {
-    extraFiles = [ dataDir ];
-    postgresqlDatabases = [ "matrix-synapse" ];
-  };
-
   nix-bitcoin.secrets.matrix-smtp-password.user = "matrix-synapse";
   # Used by dovecot2 (via the mailserver module)
   nix-bitcoin.secrets.matrix-smtp-password-hashed.user = "root";