diff --git a/servers/eisen/configuration.nix b/servers/eisen/configuration.nix index 67fd099..65b5a4f 100644 --- a/servers/eisen/configuration.nix +++ b/servers/eisen/configuration.nix @@ -106,6 +106,9 @@ in oidc_native_auth = true; ip_source = "rightmost_x_forwarded_for"; registration_token_file = "/etc/secrets/matrix-registration-token"; + database_backup_path = "/var/lib/tuwunel/database-backups"; + database_backups_to_keep = 2; + admin_signal_execute = [ "server backup-database" ]; well_known = { client = "https://${matrixHost}"; @@ -457,6 +460,62 @@ in after = [ "livekit-keys.service" ]; }; + services.restic.backups.matrix = { + repository = "sftp:restic-eisen@fern:/media/large/restic/eisen"; + passwordFile = "/etc/secrets/restic-password"; + initialize = true; + paths = [ + "/var/lib/tuwunel/database-backups" + "/var/lib/tuwunel/media" + "/var/lib/livekit/keys" + "/etc/secrets/matrix-registration-token" + ]; + extraOptions = [ + "sftp.command='${lib.getExe pkgs.tailscale} ssh restic-eisen@fern -s sftp'" + ]; + extraBackupArgs = [ "--tag=matrix" ]; + pruneOpts = [ + "--keep-hourly=24" + "--keep-daily=14" + "--keep-weekly=8" + "--keep-monthly=12" + ]; + timerConfig = { + OnCalendar = "*-*-* 09,12,15,18,21:00:00"; + Persistent = true; + RandomizedDelaySec = "20m"; + }; + backupPrepareCommand = '' + #!${pkgs.runtimeShell} + set -eu + + backup_meta=/var/lib/tuwunel/database-backups/meta + before="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)" + + ${lib.getExe' pkgs.systemd "systemctl"} kill --kill-whom=main --signal=SIGUSR2 tuwunel.service + + # RocksDB writes the metadata file only after an online backup is complete. + for ((attempt = 0; attempt < 120; attempt++)); do + after="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)" + if [[ -n "$after" && "$after" != "$before" ]]; then + exit 0 + fi + ${pkgs.coreutils}/bin/sleep 5 + done + + echo "Timed out waiting for Tuwunel's online database backup" >&2 + exit 1 + ''; + }; + + # A sleeping Fern is normal. Keep retrying without overlapping timer runs, + # and promptly abandon dead SSH connections if it sleeps during a backup. + systemd.services.restic-backups-matrix.serviceConfig = { + Restart = "on-failure"; + RestartSec = "30m"; + TimeoutStartSec = "12h"; + }; + # The Karakeep module still emits this option, but Meilisearch 1.51 removed it. systemd.services.meilisearch.serviceConfig.ExecStartPre = lib.mkAfter [ "${lib.getExe pkgs.gnused} -i '/^experimental_dumpless_upgrade =/d' \${RUNTIME_DIRECTORY}/config.toml" diff --git a/servers/fern/configuration.nix b/servers/fern/configuration.nix index f1fe3a6..be4a1e3 100644 --- a/servers/fern/configuration.nix +++ b/servers/fern/configuration.nix @@ -10,27 +10,31 @@ let system = pkgs.stdenv.hostPlatform.system; - waydroidNvidia = - (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs - (_: { - postFixup = '' - wrapProgram $out/bin/waydroid \ - --prefix PATH : ${lib.makeBinPath [ pkgs.lxc pkgs.kmod pkgs.util-linux ]} - wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \ - --prefix PATH : ${ - lib.makeBinPath [ - pkgs.lxc - pkgs.kmod - pkgs.iptables - pkgs.nftables - pkgs.iproute2 - pkgs.dnsmasq - pkgs.gawk - pkgs.getent - ] - } - ''; - }); + waydroidNvidia = (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs (_: { + postFixup = '' + wrapProgram $out/bin/waydroid \ + --prefix PATH : ${ + lib.makeBinPath [ + pkgs.lxc + pkgs.kmod + pkgs.util-linux + ] + } + wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \ + --prefix PATH : ${ + lib.makeBinPath [ + pkgs.lxc + pkgs.kmod + pkgs.iptables + pkgs.nftables + pkgs.iproute2 + pkgs.dnsmasq + pkgs.gawk + pkgs.getent + ] + } + ''; + }); ninfs = pkgs.python3Packages.buildPythonApplication { pname = "ninfs"; @@ -144,6 +148,19 @@ in # }; # ssh -R (remote port forward) to this server should listen publicly services.openssh.settings.GatewayPorts = "yes"; + services.tailscale.extraUpFlags = lib.mkAfter [ "--ssh" ]; + + users.groups.restic-eisen = { }; + users.users.restic-eisen = { + isSystemUser = true; + group = "restic-eisen"; + home = "/"; + }; + + systemd.tmpfiles.rules = [ + "d /media/large/restic 0755 root root -" + "d /media/large/restic/eisen 0700 restic-eisen restic-eisen -" + ]; boot = { # Steam client bug #13186: xpad conflicts with Steam Controller emulation # and crashes Steam while a game is starting. @@ -354,6 +371,11 @@ in }; }; + services.tailscale.extraSetFlags = lib.mkForce [ + "--advertise-exit-node" + "--ssh" + ]; + nix.optimise = { automatic = true; persistent = true; @@ -378,7 +400,7 @@ in # powerManagement.enable = true; hardware.nvidia-container-toolkit.enable = true; virtualisation.docker.daemon.settings.features.cdi = true; - + # Keep the host resolver off Waydroid's 192.168.240.1:53 listener. services.dnsmasq.settings = { listen-address = "127.0.0.1"; diff --git a/servers/ui-mode/configuration.nix b/servers/ui-mode/configuration.nix index c1fc2bd..9c5f45e 100644 --- a/servers/ui-mode/configuration.nix +++ b/servers/ui-mode/configuration.nix @@ -349,7 +349,7 @@ enable = true; useRoutingFeatures = "both"; openFirewall = true; - extraUpFlags = [ "--advertise-exit-node" ]; + extraUpFlags = lib.mkDefault [ "--advertise-exit-node" ]; }; hardware.opentabletdriver.enable = true; virtualisation.docker = {