update

servers/eisen/configuration.nix

@@ -106,6 +106,9 @@ in
         oidc_native_auth = true;
         ip_source = "rightmost_x_forwarded_for";
         registration_token_file = "/etc/secrets/matrix-registration-token";
+        database_backup_path = "/var/lib/tuwunel/database-backups";
+        database_backups_to_keep = 2;
+        admin_signal_execute = [ "server backup-database" ];
 
         well_known = {
           client = "https://${matrixHost}";
@@ -457,6 +460,62 @@ in
     after = [ "livekit-keys.service" ];
   };
 
+  services.restic.backups.matrix = {
+    repository = "sftp:restic-eisen@fern:/media/large/restic/eisen";
+    passwordFile = "/etc/secrets/restic-password";
+    initialize = true;
+    paths = [
+      "/var/lib/tuwunel/database-backups"
+      "/var/lib/tuwunel/media"
+      "/var/lib/livekit/keys"
+      "/etc/secrets/matrix-registration-token"
+    ];
+    extraOptions = [
+      "sftp.command='${lib.getExe pkgs.tailscale} ssh restic-eisen@fern -s sftp'"
+    ];
+    extraBackupArgs = [ "--tag=matrix" ];
+    pruneOpts = [
+      "--keep-hourly=24"
+      "--keep-daily=14"
+      "--keep-weekly=8"
+      "--keep-monthly=12"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 09,12,15,18,21:00:00";
+      Persistent = true;
+      RandomizedDelaySec = "20m";
+    };
+    backupPrepareCommand = ''
+      #!${pkgs.runtimeShell}
+      set -eu
+
+      backup_meta=/var/lib/tuwunel/database-backups/meta
+      before="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)"
+
+      ${lib.getExe' pkgs.systemd "systemctl"} kill --kill-whom=main --signal=SIGUSR2 tuwunel.service
+
+      # RocksDB writes the metadata file only after an online backup is complete.
+      for ((attempt = 0; attempt < 120; attempt++)); do
+        after="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)"
+        if [[ -n "$after" && "$after" != "$before" ]]; then
+          exit 0
+        fi
+        ${pkgs.coreutils}/bin/sleep 5
+      done
+
+      echo "Timed out waiting for Tuwunel's online database backup" >&2
+      exit 1
+    '';
+  };
+
+  # A sleeping Fern is normal. Keep retrying without overlapping timer runs,
+  # and promptly abandon dead SSH connections if it sleeps during a backup.
+  systemd.services.restic-backups-matrix.serviceConfig = {
+    Restart = "on-failure";
+    RestartSec = "30m";
+    TimeoutStartSec = "12h";
+  };
+
   # The Karakeep module still emits this option, but Meilisearch 1.51 removed it.
   systemd.services.meilisearch.serviceConfig.ExecStartPre = lib.mkAfter [
     "${lib.getExe pkgs.gnused} -i '/^experimental_dumpless_upgrade =/d' \${RUNTIME_DIRECTORY}/config.toml"

servers/fern/configuration.nix

@@ -10,27 +10,31 @@
 
 let
   system = pkgs.stdenv.hostPlatform.system;
-  waydroidNvidia =
+  waydroidNvidia = (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs (_: {
-    (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs
+    postFixup = ''
-      (_: {
+      wrapProgram $out/bin/waydroid \
-        postFixup = ''
+        --prefix PATH : ${
-          wrapProgram $out/bin/waydroid \
+          lib.makeBinPath [
-            --prefix PATH : ${lib.makeBinPath [ pkgs.lxc pkgs.kmod pkgs.util-linux ]}
+            pkgs.lxc
-          wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \
+            pkgs.kmod
-            --prefix PATH : ${
+            pkgs.util-linux
-              lib.makeBinPath [
+          ]
-                pkgs.lxc
+        }
-                pkgs.kmod
+      wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \
-                pkgs.iptables
+        --prefix PATH : ${
-                pkgs.nftables
+          lib.makeBinPath [
-                pkgs.iproute2
+            pkgs.lxc
-                pkgs.dnsmasq
+            pkgs.kmod
-                pkgs.gawk
+            pkgs.iptables
-                pkgs.getent
+            pkgs.nftables
-              ]
+            pkgs.iproute2
-            }
+            pkgs.dnsmasq
-        '';
+            pkgs.gawk
-      });
+            pkgs.getent
+          ]
+        }
+    '';
+  });
 
   ninfs = pkgs.python3Packages.buildPythonApplication {
     pname = "ninfs";
@@ -144,6 +148,19 @@ in
   # };
   # ssh -R (remote port forward) to this server should listen publicly
   services.openssh.settings.GatewayPorts = "yes";
+  services.tailscale.extraUpFlags = lib.mkAfter [ "--ssh" ];
+
+  users.groups.restic-eisen = { };
+  users.users.restic-eisen = {
+    isSystemUser = true;
+    group = "restic-eisen";
+    home = "/";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /media/large/restic 0755 root root -"
+    "d /media/large/restic/eisen 0700 restic-eisen restic-eisen -"
+  ];
   boot = {
     # Steam client bug #13186: xpad conflicts with Steam Controller emulation
     # and crashes Steam while a game is starting.
@@ -354,6 +371,11 @@ in
     };
   };
 
+  services.tailscale.extraSetFlags = lib.mkForce [
+    "--advertise-exit-node"
+    "--ssh"
+  ];
+
   nix.optimise = {
     automatic = true;
     persistent = true;
@@ -378,7 +400,7 @@ in
   # powerManagement.enable = true;
   hardware.nvidia-container-toolkit.enable = true;
   virtualisation.docker.daemon.settings.features.cdi = true;
-  
+
   # Keep the host resolver off Waydroid's 192.168.240.1:53 listener.
   services.dnsmasq.settings = {
     listen-address = "127.0.0.1";

servers/ui-mode/configuration.nix

@@ -349,7 +349,7 @@
     enable = true;
     useRoutingFeatures = "both";
     openFirewall = true;
-    extraUpFlags = [ "--advertise-exit-node" ];
+    extraUpFlags = lib.mkDefault [ "--advertise-exit-node" ];
   };
   hardware.opentabletdriver.enable = true;
   virtualisation.docker = {