fix colors

.config/hypr/hyprland.lua

@@ -216,6 +216,7 @@ require("dms/colors")
 require("dms/outputs")
 -- require("dms.cursor")
 -- require("dms.binds")
+-- require("dms/binds")
 require("dms/layout")
 require("dms/binds-user")
 require("dms/windowrules")

flake.nix

@@ -220,7 +220,7 @@
       # Export sysbox NixOS module for external use
       nixosModules.sysbox = import ./modules/sysbox.nix;
       nixosModules.tuwunel-admin = import ./modules/tuwunel-admin.nix;
-      #nixosModules.budkyber = import ./modules/budkyber.nix;
+      #nixosModules.adctf = import ./modules/adctf.nix;
 
       packages.x86_64-linux = rec {
         tuwunel-admin =
@@ -237,10 +237,10 @@
           paseo.nixosModules.paseo
           determinate.nixosModules.default
           home-manager.nixosModules.home-manager
-          #./modules/budkyber.nix
+          #./modules/adctf.nix
           {
             services.hypr-kdeconnect-fix.enable = true;
-            #services.budkyber.enable = true;
+            #services.adctf.enable = true;
             home-manager.extraSpecialArgs = attrs;
             home-manager.useGlobalPkgs = true;
             home-manager.useUserPackages = true;

modules/adctf.nix

@@ -0,0 +1,492 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.adctf;
+  inherit (lib)
+    concatMapStringsSep
+    escapeShellArg
+    mapAttrs'
+    mkEnableOption
+    mkIf
+    mkMerge
+    mkOption
+    nameValuePair
+    optionalAttrs
+    types
+    ;
+
+  compose = lib.getExe pkgs.docker-compose;
+  portOverlays = {
+    collector = pkgs.writeText "adctf-collector-ports.yml" ''
+      services:
+        collector:
+          ports: [ "127.0.0.1:6256:6256" ]
+    '';
+    grafana = pkgs.writeText "adctf-grafana-ports.yml" ''
+      services:
+        grafana:
+          ports: [ "127.0.0.1:6003:6003" ]
+    '';
+    loki = pkgs.writeText "adctf-loki-ports.yml" ''
+      services:
+        loki:
+          ports: [ "127.0.0.1:6004:6004" ]
+        alloy:
+          ports: [ "127.0.0.1:6005:6005" ]
+    '';
+    prometheus = pkgs.writeText "adctf-prometheus-ports.yml" ''
+      services:
+        prometheus:
+          ports: [ "127.0.0.1:9090:9090" ]
+    '';
+    cloudbeaver = pkgs.writeText "adctf-cloudbeaver-ports.yml" ''
+      services:
+        bober:
+          ports: [ "127.0.0.1:8978:8978" ]
+    '';
+  };
+
+
+  statekOverlay = pkgs.writeText "adctf-statek-overlay.yml" ''
+    services:
+      db:
+        networks:
+          statek:
+          cct: { aliases: [ statek_db ] }
+          cct6: { aliases: [ statek_db ] }
+      scoreboard:
+        networks:
+          statek:
+          cct: { aliases: [ statek_scoreboard ] }
+          cct6: { aliases: [ statek_scoreboard ] }
+      attackinfo:
+        networks:
+          statek:
+          cct: { aliases: [ statek_attackinfo ] }
+          cct6: { aliases: [ statek_attackinfo ] }
+      submitter:
+        networks:
+          statek:
+          cct: { aliases: [ statek_submitter ] }
+          cct6: { aliases: [ statek_submitter ] }
+      api:
+        networks:
+          statek:
+            aliases: [ api ]
+          cct: { aliases: [ statek_api ] }
+          cct6: { aliases: [ statek_api ] }
+      frontend:
+        networks:
+          statek:
+          cct: { aliases: [ statek_frontend ] }
+          cct6: { aliases: [ statek_frontend ] }
+    networks:
+      cct: { name: cct, external: true }
+      cct6: { name: cct6, external: true }
+  '';
+
+  tulipOverlay = pkgs.writeText "adctf-tulip-overlay.yml" ''
+    services:
+      timescale:
+        networks:
+          internal:
+          cct: { aliases: [ tulip_timescale ] }
+          cct6: { aliases: [ tulip_timescale ] }
+      frontend:
+        ports: !override [ "127.0.0.1:3000:3000" ]
+        networks:
+          internal:
+          cct: { aliases: [ tulip_frontend ] }
+          cct6: { aliases: [ tulip_frontend ] }
+      api:
+        networks:
+          internal:
+          cct: { aliases: [ tulip_api ] }
+          cct6: { aliases: [ tulip_api ] }
+      flagids:
+        networks:
+          internal:
+          cct: { aliases: [ tulip_flagids ] }
+          cct6: { aliases: [ tulip_flagids ] }
+      assembler:
+        networks:
+          internal:
+          cct: { aliases: [ tulip_assembler ] }
+          cct6: { aliases: [ tulip_assembler ] }
+      enricher:
+        networks:
+          internal:
+          cct: { aliases: [ tulip_enricher ] }
+          cct6: { aliases: [ tulip_enricher ] }
+    networks:
+      cct: { name: cct, external: true }
+      cct6: { name: cct6, external: true }
+  '';
+
+  infrastructureStacks = {
+    collector = {
+      directory = "${cfg.infrastructureRoot}/collector";
+      files = [
+        "${cfg.infrastructureRoot}/collector/docker-compose.yml"
+        portOverlays.collector
+      ];
+    };
+    grafana = {
+      directory = "${cfg.infrastructureRoot}/grafana";
+      files = [
+        "${cfg.infrastructureRoot}/grafana/docker-compose.yml"
+        portOverlays.grafana
+      ];
+    };
+    loki = {
+      directory = "${cfg.infrastructureRoot}/loki-adctf";
+      files = [
+        "${cfg.infrastructureRoot}/loki-adctf/docker-compose.yml"
+        portOverlays.loki
+      ];
+    };
+    prometheus = {
+      directory = "${cfg.infrastructureRoot}/prometheus";
+      files = [
+        "${cfg.infrastructureRoot}/prometheus/docker-compose.yml"
+        portOverlays.prometheus
+      ];
+    };
+    suricata = {
+      directory = "${cfg.infrastructureRoot}/suricata";
+      files = [ "${cfg.infrastructureRoot}/suricata/docker-compose.yml" ];
+    };
+  };
+
+  applicationStacks = {
+    statek = {
+      directory = cfg.statekRoot;
+      files = [
+        "${cfg.statekRoot}/compose.yml"
+        statekOverlay
+      ];
+    };
+    tulip = {
+      directory = cfg.tulipRoot;
+      files = [
+        "${cfg.tulipRoot}/compose.yml"
+        tulipOverlay
+      ];
+      environment = {
+        TRAFFIC_DIR_HOST = "${cfg.infrastructureRoot}/traffic";
+        TRAFFIC_DIR_DOCKER = "/traffic";
+      };
+    };
+  };
+
+  cloudbeaverStack = {
+    cloudbeaver = {
+      directory = "${cfg.infrastructureRoot}/other/cloudbeaver";
+      files = [
+        "${cfg.infrastructureRoot}/other/cloudbeaver/docker-compose.yml"
+        portOverlays.cloudbeaver
+      ];
+    };
+  };
+
+  stacks =
+    infrastructureStacks
+    // applicationStacks
+    // optionalAttrs cfg.cloudbeaver.enable cloudbeaverStack;
+
+  proxyPorts = {
+    collector = 6256;
+    grafana = 6003;
+    loki = 6004;
+    alloy = 6005;
+    prometheus = 9090;
+    statek = 5173;
+    statek-api = 8080;
+    tulip = 3000;
+  }
+  // optionalAttrs cfg.cloudbeaver.enable { cloudbeaver = 8978; };
+
+  proxyHosts = mapAttrs' (
+    name: port:
+    nameValuePair "${name}.${cfg.proxy.baseDomain}:80" {
+      extraConfig = "reverse_proxy http://127.0.0.1:${toString port}";
+    }
+  ) proxyPorts;
+
+  composeCommand = name: stack:
+    "${compose} --project-name ${escapeShellArg "adctf-${name}"} "
+    + concatMapStringsSep " " (file: "-f ${escapeShellArg file}") stack.files;
+
+  mkComposeService = name: stack:
+    nameValuePair "adctf-${name}" {
+      description = "adctf ${name} containers";
+      wantedBy = [ "multi-user.target" ];
+      requires = [
+        "docker.service"
+        "adctf-networks.service"
+      ];
+      after = [
+        "docker.service"
+        "adctf-networks.service"
+        "network-online.target"
+      ];
+      wants = [ "network-online.target" ];
+      path = [ pkgs.coreutils ];
+      environment = stack.environment or { };
+      script = ''
+        test -f ${escapeShellArg (builtins.head stack.files)}
+        ${composeCommand name stack} up --detach --build --remove-orphans
+      '';
+      preStop = ''
+        ${composeCommand name stack} down
+      '';
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        WorkingDirectory = stack.directory;
+        TimeoutStartSec = "infinity";
+        TimeoutStopSec = "5min";
+      };
+    };
+in
+{
+  options.services.adctf = {
+    enable = mkEnableOption "the adctf attack-defense infrastructure";
+
+    user = mkOption {
+      type = types.str;
+      default = "dan";
+      description = "Local user allowed to manage the container and VM runtimes.";
+    };
+
+    infrastructureRoot = mkOption {
+      type = types.str;
+      default = "/home/dan/projects/ad-infrastructure-private";
+      description = "Absolute path to the ad-infrastructure-private checkout.";
+    };
+
+    statekRoot = mkOption {
+      type = types.str;
+      default = "/home/dan/projects/statek";
+      description = "Absolute path to the Statek checkout.";
+    };
+
+    tulipRoot = mkOption {
+      type = types.str;
+      default = "/home/dan/projects/tulip-private";
+      description = "Absolute path to the Tulip checkout.";
+    };
+
+    cloudbeaver.enable = mkOption {
+      type = types.bool;
+      default = true;
+      description = "Start the optional CloudBeaver database UI.";
+    };
+
+    proxy = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Publish adctf HTTP services through Caddy.";
+      };
+
+      baseDomain = mkOption {
+        type = types.str;
+        default = "fern.danbulant.cloud";
+        description = "Base domain below which each service gets its own subdomain.";
+      };
+    };
+
+    cgroup = {
+      reserveCpuPercent = mkOption {
+        type = types.ints.between 0 99;
+        default = 50;
+        description = "Percentage of one logical CPU reserved outside docker.slice.";
+      };
+
+      reserveMemoryMiB = mkOption {
+        type = types.ints.positive;
+        default = 1024;
+        description = "Physical memory reserved outside docker.slice.";
+      };
+
+      reserveSwapMiB = mkOption {
+        type = types.ints.unsigned;
+        default = 1024;
+        description = "Swap reserved outside docker.slice when swap is available.";
+      };
+    };
+
+    virtualMachines = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Enable libvirt/QEMU management for a qcow vulnbox image.";
+      };
+
+      virtualbox.enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Enable VirtualBox as a fallback for VirtualBox-formatted vulnbox images.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable (mkMerge [
+    {
+      assertions = [
+        {
+          assertion = lib.hasPrefix "/" cfg.infrastructureRoot;
+          message = "services.adctf.infrastructureRoot must be an absolute path";
+        }
+        {
+          assertion = lib.hasPrefix "/" cfg.statekRoot;
+          message = "services.adctf.statekRoot must be an absolute path";
+        }
+        {
+          assertion = lib.hasPrefix "/" cfg.tulipRoot;
+          message = "services.adctf.tulipRoot must be an absolute path";
+        }
+      ];
+
+      virtualisation.docker = {
+        enable = true;
+        daemon.settings."cgroup-parent" = "docker.slice";
+      };
+
+      environment.systemPackages = [
+        pkgs.docker-compose
+      ];
+
+      systemd.tmpfiles.rules = [
+        "d ${cfg.infrastructureRoot}/traffic 0775 ${cfg.user} users -"
+      ];
+
+      users.users.${cfg.user}.extraGroups = [ "docker" ];
+
+      systemd.slices.docker = {
+        description = "Docker container resource budget";
+        sliceConfig = {
+          CPUAccounting = true;
+          MemoryAccounting = true;
+        };
+      };
+
+      systemd.services = {
+        docker = {
+          requires = [ "adctf-cgroup-limits.service" ];
+          after = [ "adctf-cgroup-limits.service" ];
+          wantedBy = [ "multi-user.target" ];
+        };
+
+        adctf-cgroup-limits = {
+          description = "Reserve host resources outside docker.slice";
+          before = [ "docker.service" ];
+          path = [
+            pkgs.coreutils
+            pkgs.systemd
+          ];
+          script = ''
+            logical_cpus="$(nproc --all)"
+            cpu_quota="$((logical_cpus * 100 - ${toString cfg.cgroup.reserveCpuPercent}))"
+            memory_total_kib=0
+            swap_total_kib=0
+
+            while read -r key value _; do
+              case "$key" in
+                MemTotal:) memory_total_kib="$value" ;;
+                SwapTotal:) swap_total_kib="$value" ;;
+              esac
+            done < /proc/meminfo
+
+            reserve_memory_kib=$((${toString cfg.cgroup.reserveMemoryMiB} * 1024))
+            if ((memory_total_kib <= reserve_memory_kib)); then
+              echo "Cannot reserve ${toString cfg.cgroup.reserveMemoryMiB} MiB from $((memory_total_kib / 1024)) MiB of physical memory" >&2
+              exit 1
+            fi
+            memory_max_kib="$((memory_total_kib - reserve_memory_kib))"
+
+            reserve_swap_kib=$((${toString cfg.cgroup.reserveSwapMiB} * 1024))
+            if ((swap_total_kib > reserve_swap_kib)); then
+              swap_max_kib="$((swap_total_kib - reserve_swap_kib))"
+            else
+              swap_max_kib=0
+            fi
+
+            systemctl set-property --runtime docker.slice \
+              CPUQuota="''${cpu_quota}%" \
+              MemoryMax="''${memory_max_kib}K" \
+              MemorySwapMax="''${swap_max_kib}K"
+          '';
+          preStop = ''
+            systemctl set-property --runtime docker.slice \
+              CPUQuota=infinity MemoryMax=infinity MemorySwapMax=infinity
+          '';
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+          };
+        };
+
+        adctf-networks = {
+          description = "Create adctf container networks";
+          requires = [ "docker.service" ];
+          after = [ "docker.service" ];
+          path = [ config.virtualisation.docker.package ];
+          script = ''
+            docker network inspect cct >/dev/null 2>&1 || \
+              docker network create \
+                --gateway 10.66.0.1 \
+                --ip-range 10.66.0.0/16 \
+                --subnet 10.66.0.0/16 \
+                cct
+
+            docker network inspect cct6 >/dev/null 2>&1 || \
+              docker network create \
+                --ipv6 \
+                --subnet 2001:db8:1::/64 \
+                cct6
+          '';
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+          };
+        };
+      }
+      // mapAttrs' mkComposeService stacks;
+
+      services.caddy = mkIf cfg.proxy.enable {
+        enable = true;
+        virtualHosts = proxyHosts;
+      };
+    }
+
+    (mkIf cfg.virtualMachines.enable {
+      virtualisation.libvirtd.enable = true;
+      programs.virt-manager.enable = true;
+      users.users.${cfg.user}.extraGroups = [
+        "kvm"
+        "libvirtd"
+      ];
+      environment.systemPackages = with pkgs; [
+        qemu
+        quickemu
+        virt-viewer
+      ];
+    })
+
+    (mkIf cfg.virtualMachines.virtualbox.enable {
+      virtualisation.virtualbox.host = {
+        enable = true;
+        enableKvm = true;
+        addNetworkInterface = false;
+      };
+      users.users.${cfg.user}.extraGroups = [ "vboxusers" ];
+    })
+  ]);
+}

servers/eisen/configuration.nix

@@ -106,6 +106,9 @@ in
         oidc_native_auth = true;
         ip_source = "rightmost_x_forwarded_for";
         registration_token_file = "/etc/secrets/matrix-registration-token";
+        database_backup_path = "/var/lib/tuwunel/database-backups";
+        database_backups_to_keep = 2;
+        admin_signal_execute = [ "server backup-database" ];
 
         well_known = {
           client = "https://${matrixHost}";
@@ -457,6 +460,62 @@ in
     after = [ "livekit-keys.service" ];
   };
 
+  services.restic.backups.matrix = {
+    repository = "sftp:restic-eisen@fern:/media/large/restic/eisen";
+    passwordFile = "/etc/secrets/restic-password";
+    initialize = true;
+    paths = [
+      "/var/lib/tuwunel/database-backups"
+      "/var/lib/tuwunel/media"
+      "/var/lib/livekit/keys"
+      "/etc/secrets/matrix-registration-token"
+    ];
+    extraOptions = [
+      "sftp.command='${lib.getExe pkgs.tailscale} ssh restic-eisen@fern -s sftp'"
+    ];
+    extraBackupArgs = [ "--tag=matrix" ];
+    pruneOpts = [
+      "--keep-hourly=24"
+      "--keep-daily=14"
+      "--keep-weekly=8"
+      "--keep-monthly=12"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 09,12,15,18,21:00:00";
+      Persistent = true;
+      RandomizedDelaySec = "20m";
+    };
+    backupPrepareCommand = ''
+      #!${pkgs.runtimeShell}
+      set -eu
+
+      backup_meta=/var/lib/tuwunel/database-backups/meta
+      before="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)"
+
+      ${lib.getExe' pkgs.systemd "systemctl"} kill --kill-whom=main --signal=SIGUSR2 tuwunel.service
+
+      # RocksDB writes the metadata file only after an online backup is complete.
+      for ((attempt = 0; attempt < 120; attempt++)); do
+        after="$(${pkgs.findutils}/bin/find "$backup_meta" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | ${pkgs.coreutils}/bin/sort || true)"
+        if [[ -n "$after" && "$after" != "$before" ]]; then
+          exit 0
+        fi
+        ${pkgs.coreutils}/bin/sleep 5
+      done
+
+      echo "Timed out waiting for Tuwunel's online database backup" >&2
+      exit 1
+    '';
+  };
+
+  # A sleeping Fern is normal. Keep retrying without overlapping timer runs,
+  # and promptly abandon dead SSH connections if it sleeps during a backup.
+  systemd.services.restic-backups-matrix.serviceConfig = {
+    Restart = "on-failure";
+    RestartSec = "30m";
+    TimeoutStartSec = "12h";
+  };
+
   # The Karakeep module still emits this option, but Meilisearch 1.51 removed it.
   systemd.services.meilisearch.serviceConfig.ExecStartPre = lib.mkAfter [
     "${lib.getExe pkgs.gnused} -i '/^experimental_dumpless_upgrade =/d' \${RUNTIME_DIRECTORY}/config.toml"

servers/fern/configuration.nix

@@ -10,27 +10,31 @@
 
 let
   system = pkgs.stdenv.hostPlatform.system;
-  waydroidNvidia =
-    (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs
-      (_: {
-        postFixup = ''
-          wrapProgram $out/bin/waydroid \
-            --prefix PATH : ${lib.makeBinPath [ pkgs.lxc pkgs.kmod pkgs.util-linux ]}
-          wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \
-            --prefix PATH : ${
-              lib.makeBinPath [
-                pkgs.lxc
-                pkgs.kmod
-                pkgs.iptables
-                pkgs.nftables
-                pkgs.iproute2
-                pkgs.dnsmasq
-                pkgs.gawk
-                pkgs.getent
-              ]
-            }
-        '';
-      });
+  waydroidNvidia = (waydroid-nvidia-nix.packages.${system}.waydroid-nvidia-full).overrideAttrs (_: {
+    postFixup = ''
+      wrapProgram $out/bin/waydroid \
+        --prefix PATH : ${
+          lib.makeBinPath [
+            pkgs.lxc
+            pkgs.kmod
+            pkgs.util-linux
+          ]
+        }
+      wrapProgram $out/lib/waydroid/data/scripts/waydroid-net.sh \
+        --prefix PATH : ${
+          lib.makeBinPath [
+            pkgs.lxc
+            pkgs.kmod
+            pkgs.iptables
+            pkgs.nftables
+            pkgs.iproute2
+            pkgs.dnsmasq
+            pkgs.gawk
+            pkgs.getent
+          ]
+        }
+    '';
+  });
 
   ninfs = pkgs.python3Packages.buildPythonApplication {
     pname = "ninfs";
@@ -144,6 +148,19 @@ in
   # };
   # ssh -R (remote port forward) to this server should listen publicly
   services.openssh.settings.GatewayPorts = "yes";
+  services.tailscale.extraUpFlags = lib.mkAfter [ "--ssh" ];
+
+  users.groups.restic-eisen = { };
+  users.users.restic-eisen = {
+    isSystemUser = true;
+    group = "restic-eisen";
+    home = "/";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /media/large/restic 0755 root root -"
+    "d /media/large/restic/eisen 0700 restic-eisen restic-eisen -"
+  ];
   boot = {
     # Steam client bug #13186: xpad conflicts with Steam Controller emulation
     # and crashes Steam while a game is starting.
@@ -354,6 +371,11 @@ in
     };
   };
 
+  services.tailscale.extraSetFlags = lib.mkForce [
+    "--advertise-exit-node"
+    "--ssh"
+  ];
+
   nix.optimise = {
     automatic = true;
     persistent = true;
@@ -378,7 +400,7 @@ in
   # powerManagement.enable = true;
   hardware.nvidia-container-toolkit.enable = true;
   virtualisation.docker.daemon.settings.features.cdi = true;
-  
+
   # Keep the host resolver off Waydroid's 192.168.240.1:53 listener.
   services.dnsmasq.settings = {
     listen-address = "127.0.0.1";

servers/ui-mode/configuration.nix

@@ -349,7 +349,7 @@
     enable = true;
     useRoutingFeatures = "both";
     openFirewall = true;
-    extraUpFlags = [ "--advertise-exit-node" ];
+    extraUpFlags = lib.mkDefault [ "--advertise-exit-node" ];
   };
   hardware.opentabletdriver.enable = true;
   virtualisation.docker = {