mirror of
https://github.com/danbulant/dotfiles
synced 2026-06-24 01:01:49 +00:00
Compare commits
6 commits
9407985eb3
...
51d7fe8633
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
51d7fe8633 | ||
|
|
847fe42a9c | ||
|
|
73fdccfc2e | ||
|
|
45e4b15be5 | ||
|
|
e4314f8c18 | ||
|
|
12c35ba942 |
10 changed files with 1502 additions and 26 deletions
|
|
@ -69,13 +69,33 @@ plugin {
|
|||
|
||||
# See https://wiki.hyprland.org/Configuring/Keywords/ for more
|
||||
|
||||
env = QT_QPA_PLATFORMTHEME,qt5ct
|
||||
env = QT_AUTO_SCREEN_SCALE_FACTOR,1
|
||||
env = QT_ENABLE_HIGHDPI_SCALING,1
|
||||
env = QT_QPA_PLATFORM,wayland;xcb
|
||||
env = XCURSOR_SIZE,24
|
||||
env = LIBVA_DRIVER_NAME,nvidia
|
||||
env = XDG_SESSION_TYPE,wayland
|
||||
env = __GLX_VENDOR_LIBRARY_NAME,nvidia
|
||||
#env = WLR_NO_HARDWARE_CURSORS,1
|
||||
env = WLR_DRM_DEVICES,/dev/dri/card1:/dev/dri/card0
|
||||
env = GDK_SCALE,1
|
||||
env = GDK_BACKEND,wayland,x11
|
||||
env = QT_WAYLAND_DISABLE_WINDOWDECORATION,1
|
||||
env = MOZ_ENABLE_WAYLAND,1
|
||||
env = XDG_SESSION_DESKTOP,Hyprland
|
||||
env = XDG_CURRENT_DESKTOP,Hyprland
|
||||
env = QT_ENABLE_FONTCONFIG_CACHE,1
|
||||
|
||||
# Execute your favorite apps at launch
|
||||
# exec-once = waybar & hyprpaper & firefox
|
||||
|
||||
# Import the completed Hyprland session environment before launching long-lived helpers.
|
||||
exec-once=dbus-update-activation-environment --systemd --all
|
||||
exec-once=systemctl --user import-environment WAYLAND_DISPLAY DISPLAY XDG_CURRENT_DESKTOP XDG_SESSION_DESKTOP XDG_SESSION_TYPE WLR_DRM_DEVICES LIBVA_DRIVER_NAME __GLX_VENDOR_LIBRARY_NAME QT_QPA_PLATFORM QT_QPA_PLATFORMTHEME GDK_BACKEND MOZ_ENABLE_WAYLAND PATH XDG_DATA_DIRS XDG_CONFIG_DIRS DBUS_SESSION_BUS_ADDRESS
|
||||
|
||||
# KDE auth agent
|
||||
|
||||
exec-once=dbus-update-activation-environment --systemd --all
|
||||
exec-once=systemctl --user import-environment WAYLAND_DISPLAY XDG_CURRENT_DESKTOP QT_QPA_PLATFORMTHEME
|
||||
# exec-once=/usr/lib/polkit-kde-authentication-agent-1
|
||||
exec-once = systemctl --user start hyprpolkitagent
|
||||
exec-once=otd-daemon
|
||||
|
|
@ -103,9 +123,9 @@ exec-once = wl-paste --type text --watch cliphist store
|
|||
exec-once = wl-paste --type image --watch cliphist store
|
||||
exec-once = fcitx5
|
||||
exec-once = easyeffects --gapplication-service
|
||||
exec-once = dms run
|
||||
exec-once = voxtype daemon
|
||||
exec-once = openrgb --startminimized
|
||||
exec-once = dms run
|
||||
|
||||
#$swaylock = swaylock --screenshots --clock --indicator --effect-blur 6x6 --fade-in 0.2 --ring-color 4e9dc2 --key-hl-color 71b0ce
|
||||
$swaylock = hyprlock
|
||||
|
|
@ -115,24 +135,6 @@ $swaylock = hyprlock
|
|||
# Needs repackaging...
|
||||
# exec-once=/usr/lib/pam_kwallet_init
|
||||
|
||||
env = QT_QPA_PLATFORMTHEME,qt5ct
|
||||
env = QT_AUTO_SCREEN_SCALE_FACTOR,1
|
||||
env = QT_ENABLE_HIGHDPI_SCALING,1
|
||||
env = QT_QPA_PLATFORM,wayland;xcb
|
||||
env = XCURSOR_SIZE,24
|
||||
env = LIBVA_DRIVER_NAME,nvidia
|
||||
env = XDG_SESSION_TYPE,wayland
|
||||
env = __GLX_VENDOR_LIBRARY_NAME,nvidia
|
||||
#env = WLR_NO_HARDWARE_CURSORS,1
|
||||
env = WLR_DRM_DEVICES,/dev/dri/card1:/dev/dri/card0
|
||||
env = GDK_SCALE,1
|
||||
env = GDK_BACKEND,wayland,x11
|
||||
env = QT_WAYLAND_DISABLE_WINDOWDECORATION,1
|
||||
env = MOZ_ENABLE_WAYLAND,1
|
||||
env = XDG_SESSION_DESKTOP,Hyprland
|
||||
env = XDG_CURRENT_DESKTOP,Hyprland
|
||||
env = QT_ENABLE_FONTCONFIG_CACHE,1
|
||||
|
||||
#monitor=eDP-1,highrr,0x0,1.25
|
||||
#monitor=eDP-2,highrr,0x0,1.25
|
||||
#monitor=desc:AOC 24G2W1G4 0x0000297D,highrr,auto,1
|
||||
|
|
@ -362,8 +364,8 @@ bind=SUPER,A,swapactiveworkspaces,current +1
|
|||
bindle=, XF86MonBrightnessUp, exec, xbacklight -inc 10
|
||||
bindle=, XF86MonBrightnessDown, exec, xbacklight -dec 10
|
||||
|
||||
bindle=, XF86AudioRaiseVolume, exec, wpctl set-volume @DEFAULT_SINK@ 5%+;canberra-gtk-play -i audio-volume-change -d "volumeChange"
|
||||
bindle=, XF86AudioLowerVolume, exec, wpctl set-volume @DEFAULT_SINK@ 5%-;canberra-gtk-play -i audio-volume-change -d "volumeChange"
|
||||
# bindle=, XF86AudioRaiseVolume, exec, wpctl set-volume @DEFAULT_SINK@ 5%+;canberra-gtk-play -i audio-volume-change -d "volumeChange"
|
||||
# bindle=, XF86AudioLowerVolume, exec, wpctl set-volume @DEFAULT_SINK@ 5%-;canberra-gtk-play -i audio-volume-change -d "volumeChange"
|
||||
bindl=, XF86AudioMute, exec, wpctl set-mute @DEFAULT_SINK@ toggle
|
||||
bindl=, XF86AudioMicMute, exec, wpctl set-mute @DEFAULT_SOURCE@ toggle
|
||||
|
||||
|
|
@ -452,6 +454,8 @@ bind = CTRL+ALT,2,pass,^(com\.obsproject\.Studio)$
|
|||
#windowrulev2=opacity 0.9,class:Code
|
||||
#windowrulev2=opacity 0.9,class:Spotify
|
||||
|
||||
windowrulev2 = suppress_event maximize,class:^(vesktop)$
|
||||
|
||||
#windowrulev2=workspace 10,class:Code # Open Code on secondary monitor
|
||||
|
||||
#windowrulev2 = float,class:^(qt5ct)$
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
[preferred]
|
||||
default = hyprland;gtk
|
||||
org.freedesktop.impl.portal.FileChooser = kde
|
||||
org.freedesktop.impl.portal.FileChooser = gtk
|
||||
org.freedesktop.impl.portal.ScreenCast = hyprland
|
||||
org.freedesktop.impl.portal.RemoteDesktop = hypr-kdeconnect
|
||||
|
|
|
|||
3
analysis/.gitignore
vendored
Normal file
3
analysis/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
*.json
|
||||
*.csv
|
||||
__pycache__
|
||||
902
analysis/collect_network_libraries.py
Normal file
902
analysis/collect_network_libraries.py
Normal file
|
|
@ -0,0 +1,902 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Collect network-facing Nix package/library dependency metadata for fern/eisen.
|
||||
|
||||
The script intentionally starts from explicit service-facing roots instead of the
|
||||
full NixOS closure. The full closure includes desktop/session packages and base
|
||||
system plumbing that are not meaningfully "reachable through network".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
OUT = REPO / "analysis"
|
||||
HTTP_TIMEOUT = 8
|
||||
|
||||
|
||||
# Higher numbers are processed first. These are the Internet/LAN/Tailscale-facing
|
||||
# services and containers configured by servers/fern and servers/eisen.
|
||||
ROOTS = [
|
||||
(100, "fern", "service", "caddy", "config.services.caddy.package"),
|
||||
(98, "fern", "service", "openssh", "config.programs.ssh.package"),
|
||||
(97, "fern", "service", "llama-swap", "config.services.llama-swap.package"),
|
||||
(96, "fern", "service", "llama-cpp-server", "pkgs.llama-cpp"),
|
||||
(94, "fern", "service", "nix-serve", "config.services.nix-serve.package"),
|
||||
(92, "fern", "service", "steam-network-runtime", "config.programs.steam.package"),
|
||||
(90, "fern", "service", "kdeconnect", "pkgs.kdePackages.kdeconnect-kde"),
|
||||
(88, "fern", "service", "openrgb", "config.services.hardware.openrgb.package"),
|
||||
(86, "fern", "service", "docker", "config.virtualisation.docker.package"),
|
||||
(100, "eisen", "service", "caddy", "config.services.caddy.package"),
|
||||
(99, "eisen", "service", "tailscale", "config.services.tailscale.package"),
|
||||
(98, "eisen", "service", "openssh", "config.programs.ssh.package"),
|
||||
(97, "eisen", "service", "jellyfin", "config.services.jellyfin.package"),
|
||||
(96, "eisen", "service", "sonarr", "config.services.sonarr.package"),
|
||||
(95, "eisen", "service", "radarr", "config.services.radarr.package"),
|
||||
(94, "eisen", "service", "prowlarr", "config.services.prowlarr.package"),
|
||||
(93, "eisen", "service", "karakeep", "config.services.karakeep.package"),
|
||||
(92, "eisen", "service", "uptime-kuma", "config.services.uptime-kuma.package"),
|
||||
(91, "eisen", "service", "grafana", "config.services.grafana.package"),
|
||||
(90, "eisen", "service", "prometheus", "config.services.prometheus.package"),
|
||||
(89, "eisen", "service", "prometheus-node-exporter", "pkgs.prometheus-node-exporter"),
|
||||
(88, "eisen", "service", "exportarr-sonarr", "pkgs.exportarr"),
|
||||
(87, "eisen", "service", "exportarr-radarr", "pkgs.exportarr"),
|
||||
(86, "eisen", "service", "exportarr-prowlarr", "pkgs.exportarr"),
|
||||
(85, "eisen", "service", "glance", "config.services.glance.package"),
|
||||
(84, "eisen", "service", "dnsmasq", "config.services.dnsmasq.package"),
|
||||
(83, "eisen", "service", "docker", "config.virtualisation.docker.package"),
|
||||
(82, "eisen", "service", "llama-swap-exporter", "pkgs.callPackage ./servers/eisen/llama-swap-exporter/default.nix { }"),
|
||||
]
|
||||
|
||||
CONTAINER_ROOTS = [
|
||||
(80, "eisen", "container", "gluetun", "qmcgaw/gluetun"),
|
||||
(79, "eisen", "container", "qbittorrent", "lscr.io/linuxserver/qbittorrent"),
|
||||
(78, "eisen", "container", "jackett", "lscr.io/linuxserver/jackett"),
|
||||
(77, "eisen", "container", "prometheus-qb", "ghcr.io/esanchezm/prometheus-qbittorrent-exporter"),
|
||||
(76, "eisen", "container", "tolgee", "tolgee/tolgee"),
|
||||
]
|
||||
|
||||
GITHUB_RE = re.compile(r"github\.com[:/](?P<owner>[^/]+)/(?P<repo>[^/#?]+?)(?:\.git|/|#|\?|$)")
|
||||
STORE_HASH_PREFIX_RE = re.compile(r"^[0-9a-z]{32}-(?P<name>.+)$")
|
||||
|
||||
COMMON_UPSTREAMS = {
|
||||
"acl": ("https://git.savannah.nongnu.org/cgit/acl.git", "C"),
|
||||
"attr": ("https://git.savannah.nongnu.org/cgit/attr.git", "C"),
|
||||
"avahi": ("https://github.com/avahi/avahi", "C"),
|
||||
"bluez": ("https://git.kernel.org/pub/scm/bluetooth/bluez.git", "C"),
|
||||
"bzip2": ("https://sourceware.org/git/bzip2.git", "C"),
|
||||
"curl": ("https://github.com/curl/curl", "C"),
|
||||
"dbus": ("https://gitlab.freedesktop.org/dbus/dbus", "C"),
|
||||
"double-conversion": ("https://github.com/google/double-conversion", "C++"),
|
||||
"ffmpeg": ("https://git.ffmpeg.org/ffmpeg.git", "C"),
|
||||
"fuse": ("https://github.com/libfuse/libfuse", "C"),
|
||||
"glib": ("https://gitlab.gnome.org/GNOME/glib", "C"),
|
||||
"glibc": ("https://sourceware.org/git/glibc.git", "C"),
|
||||
"graphviz": ("https://gitlab.com/graphviz/graphviz", "C"),
|
||||
"gtk+3": ("https://gitlab.gnome.org/GNOME/gtk", "C"),
|
||||
"libarchive": ("https://github.com/libarchive/libarchive", "C"),
|
||||
"libbpf": ("https://github.com/libbpf/libbpf", "C"),
|
||||
"libbsd": ("https://gitlab.freedesktop.org/libbsd/libbsd", "C"),
|
||||
"libcbor": ("https://github.com/PJK/libcbor", "C"),
|
||||
"libedit": ("https://www.thrysoee.dk/editline/", "C"),
|
||||
"libfido2": ("https://github.com/Yubico/libfido2", "C"),
|
||||
"libmnl": ("https://git.netfilter.org/libmnl", "C"),
|
||||
"libnftnl": ("https://git.netfilter.org/libnftnl", "C"),
|
||||
"libpcap": ("https://github.com/the-tcpdump-group/libpcap", "C"),
|
||||
"libuv": ("https://github.com/libuv/libuv", "C"),
|
||||
"libxml2": ("https://gitlab.gnome.org/GNOME/libxml2", "C"),
|
||||
"libxslt": ("https://gitlab.gnome.org/GNOME/libxslt", "C"),
|
||||
"ncurses": ("https://invisible-island.net/ncurses/", "C"),
|
||||
"oniguruma": ("https://github.com/kkos/oniguruma", "C"),
|
||||
"openssl": ("https://github.com/openssl/openssl", "C"),
|
||||
"pcre2": ("https://github.com/PCRE2Project/pcre2", "C"),
|
||||
"pcsclite": ("https://pcsclite.apdu.fr/", "C"),
|
||||
"rhash": ("https://github.com/rhash/RHash", "C"),
|
||||
"sqlite": ("https://sqlite.org/src", "C"),
|
||||
"systemd": ("https://github.com/systemd/systemd", "C"),
|
||||
"xz": ("https://git.tukaani.org/xz.git", "C"),
|
||||
"zlib": ("https://github.com/madler/zlib", "C"),
|
||||
}
|
||||
|
||||
NUGET_NAME_PREFIXES = (
|
||||
"AngleSharp",
|
||||
"AspNetCore",
|
||||
"Azure.",
|
||||
"BouncyCastle",
|
||||
"Castle.",
|
||||
"Dapper",
|
||||
"DryIoc",
|
||||
"Fluent",
|
||||
"HarfBuzzSharp",
|
||||
"ICU4N",
|
||||
"Jellyfin.",
|
||||
"MailKit",
|
||||
"MetaBrainz.",
|
||||
"Microsoft.",
|
||||
"Mono.",
|
||||
"NETStandard.",
|
||||
"Newtonsoft.",
|
||||
"NLog",
|
||||
"NodaTime",
|
||||
"NuGet.",
|
||||
"NUnit",
|
||||
"RestSharp",
|
||||
"Serilog",
|
||||
"Servarr.",
|
||||
"SkiaSharp",
|
||||
"SQLitePCLRaw",
|
||||
"StyleCop.",
|
||||
"System.",
|
||||
"runtime.",
|
||||
)
|
||||
|
||||
NUGET_REPO_OVERRIDES = {
|
||||
"AngleSharp": "https://github.com/AngleSharp/AngleSharp",
|
||||
"AngleSharp.Xml": "https://github.com/AngleSharp/AngleSharp.Xml",
|
||||
"BitFaster.Caching": "https://github.com/bitfaster/BitFaster.Caching",
|
||||
"BlurHashSharp": "https://github.com/MarkusPalcer/BlurHashSharp",
|
||||
"BlurHashSharp.SkiaSharp": "https://github.com/MarkusPalcer/BlurHashSharp",
|
||||
"BouncyCastle.Cryptography": "https://github.com/bcgit/bc-csharp",
|
||||
"Castle.Core": "https://github.com/castleproject/Core",
|
||||
"Dapper": "https://github.com/DapperLib/Dapper",
|
||||
"DryIoc.dll": "https://github.com/dadhi/DryIoc",
|
||||
"DryIoc.Microsoft.DependencyInjection": "https://github.com/dadhi/DryIoc",
|
||||
"FluentAssertions": "https://github.com/fluentassertions/fluentassertions",
|
||||
"FluentMigrator": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentMigrator.Abstractions": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentMigrator.Extensions.Postgres": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentMigrator.Runner.Core": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentMigrator.Runner.Postgres": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentMigrator.Runner.SQLite": "https://github.com/fluentmigrator/fluentmigrator",
|
||||
"FluentValidation": "https://github.com/FluentValidation/FluentValidation",
|
||||
"HarfBuzzSharp": "https://github.com/mono/SkiaSharp",
|
||||
"HarfBuzzSharp.NativeAssets.Linux": "https://github.com/mono/SkiaSharp",
|
||||
"HarfBuzzSharp.NativeAssets.Win32": "https://github.com/mono/SkiaSharp",
|
||||
"HarfBuzzSharp.NativeAssets.macOS": "https://github.com/mono/SkiaSharp",
|
||||
"ICU4N": "https://github.com/NightOwl888/ICU4N",
|
||||
"ICU4N.Transliterator": "https://github.com/NightOwl888/ICU4N",
|
||||
"MailKit": "https://github.com/jstedfast/MailKit",
|
||||
"MetaBrainz.Common": "https://github.com/Zastai/MetaBrainz.Common",
|
||||
"MetaBrainz.Common.Json": "https://github.com/Zastai/MetaBrainz.Common.Json",
|
||||
"MetaBrainz.MusicBrainz": "https://github.com/Zastai/MetaBrainz.MusicBrainz",
|
||||
"Microsoft.Data.SqlClient": "https://github.com/dotnet/SqlClient",
|
||||
"Microsoft.Data.SqlClient.SNI.runtime": "https://github.com/dotnet/SqlClient",
|
||||
"Microsoft.Data.Sqlite": "https://github.com/dotnet/efcore",
|
||||
"Microsoft.Data.Sqlite.Core": "https://github.com/dotnet/efcore",
|
||||
"Newtonsoft.Json": "https://github.com/JamesNK/Newtonsoft.Json",
|
||||
"NLog": "https://github.com/NLog/NLog",
|
||||
"NodaTime": "https://github.com/nodatime/nodatime",
|
||||
"NUnit": "https://github.com/nunit/nunit",
|
||||
"NUnit3TestAdapter": "https://github.com/nunit/nunit3-vs-adapter",
|
||||
"RestSharp": "https://github.com/restsharp/RestSharp",
|
||||
"RestSharp.Serializers.SystemTextJson": "https://github.com/restsharp/RestSharp",
|
||||
"Sentry": "https://github.com/getsentry/sentry-dotnet",
|
||||
"Serilog": "https://github.com/serilog/serilog",
|
||||
"SkiaSharp": "https://github.com/mono/SkiaSharp",
|
||||
"SkiaSharp.HarfBuzz": "https://github.com/mono/SkiaSharp",
|
||||
"SkiaSharp.NativeAssets.Linux": "https://github.com/mono/SkiaSharp",
|
||||
"SkiaSharp.NativeAssets.Win32": "https://github.com/mono/SkiaSharp",
|
||||
"SkiaSharp.NativeAssets.macOS": "https://github.com/mono/SkiaSharp",
|
||||
"SQLitePCLRaw.bundle_e_sqlite3": "https://github.com/ericsink/SQLitePCL.raw",
|
||||
"SQLitePCLRaw.core": "https://github.com/ericsink/SQLitePCL.raw",
|
||||
"SQLitePCLRaw.lib.e_sqlite3": "https://github.com/ericsink/SQLitePCL.raw",
|
||||
"SQLitePCLRaw.provider.e_sqlite3": "https://github.com/ericsink/SQLitePCL.raw",
|
||||
"StyleCop.Analyzers": "https://github.com/DotNetAnalyzers/StyleCopAnalyzers",
|
||||
}
|
||||
|
||||
|
||||
def run(cmd: list[str], *, timeout: int = 120) -> str:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"command failed: {' '.join(cmd)}\n{proc.stderr}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, data: dict[str, Any]) -> None:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def nix_string(s: str) -> str:
|
||||
return json.dumps(s)
|
||||
|
||||
|
||||
def root_expr() -> str:
|
||||
rows = []
|
||||
for priority, host, kind, name, expr in ROOTS:
|
||||
cfg = "flake.nixosConfigurations.fern" if host == "fern" else "flake.colmenaHive.nodes.eisen"
|
||||
rows.append(
|
||||
"(let node = "
|
||||
+ cfg
|
||||
+ "; config = node.config; pkgs = node.pkgs; pkg = "
|
||||
+ expr
|
||||
+ "; in mkRoot "
|
||||
+ str(priority)
|
||||
+ " "
|
||||
+ nix_string(host)
|
||||
+ " "
|
||||
+ nix_string(kind)
|
||||
+ " "
|
||||
+ nix_string(name)
|
||||
+ " pkg)"
|
||||
)
|
||||
|
||||
return """
|
||||
let
|
||||
flake = builtins.getFlake (toString ./.);
|
||||
clean = s: builtins.unsafeDiscardStringContext (toString s);
|
||||
listOrNull = x: if builtins.isList x then map clean x else if x == null then [] else [ (clean x) ];
|
||||
mkRoot = priority: host: kind: rootName: pkg: {
|
||||
inherit priority host kind rootName;
|
||||
packageName = pkg.name or rootName;
|
||||
pname = pkg.pname or null;
|
||||
version = pkg.version or null;
|
||||
storePath = clean pkg;
|
||||
drv = if pkg ? drvPath then clean pkg.drvPath else null;
|
||||
homepage = pkg.meta.homepage or null;
|
||||
description = pkg.meta.description or null;
|
||||
sourceUrls = listOrNull (pkg.src.urls or (pkg.src.url or null));
|
||||
};
|
||||
in [
|
||||
""" + "\n".join(rows) + "\n]"
|
||||
|
||||
|
||||
def eval_roots() -> list[dict[str, Any]]:
|
||||
data = run(["nix", "eval", "--impure", "--json", "--expr", root_expr()], timeout=240)
|
||||
roots = json.loads(data)
|
||||
for priority, host, kind, name, image in CONTAINER_ROOTS:
|
||||
roots.append(
|
||||
{
|
||||
"priority": priority,
|
||||
"host": host,
|
||||
"kind": kind,
|
||||
"rootName": name,
|
||||
"packageName": image,
|
||||
"pname": name,
|
||||
"version": None,
|
||||
"storePath": None,
|
||||
"drv": None,
|
||||
"homepage": None,
|
||||
"description": "OCI image configured in virtualisation.oci-containers",
|
||||
"sourceUrls": [],
|
||||
"image": image,
|
||||
}
|
||||
)
|
||||
return sorted(roots, key=lambda r: (-int(r["priority"]), r["host"], r["rootName"]))
|
||||
|
||||
|
||||
def derivation_show_recursive(drv: str) -> dict[str, Any]:
|
||||
data = run(["nix", "derivation", "show", "-r", drv], timeout=300)
|
||||
parsed = json.loads(data)
|
||||
# Nix 2.30+ returns {"version": 3, "derivations": {"basename.drv": ...}};
|
||||
# older Nix returned {"/nix/store/...drv": ...}. Normalize to basename keys.
|
||||
derivations = parsed.get("derivations") if isinstance(parsed, dict) else None
|
||||
if isinstance(derivations, dict):
|
||||
return derivations
|
||||
return {Path(k).name: v for k, v in parsed.items()}
|
||||
|
||||
|
||||
def drv_meta(drv: str, all_drvs: dict[str, Any]) -> dict[str, Any]:
|
||||
item = all_drvs.get(Path(drv).name, all_drvs.get(drv, {}))
|
||||
env = item.get("env", {})
|
||||
name = clean_library_name(env.get("pname") or env.get("name") or Path(drv).name.removesuffix(".drv"))
|
||||
return {
|
||||
"name": name,
|
||||
"version": env.get("version"),
|
||||
"homepage": env.get("homepage") or env.get("meta.homepage"),
|
||||
"description": env.get("meta.description") or env.get("description"),
|
||||
"source_link": source_from_env(env),
|
||||
"language": infer_language(name, env),
|
||||
}
|
||||
|
||||
|
||||
def clean_library_name(name: str) -> str:
|
||||
match = STORE_HASH_PREFIX_RE.match(name)
|
||||
if match:
|
||||
name = match.group("name")
|
||||
for suffix in (".nupkg", ".tar.gz", ".tar.xz", ".zip", ".drv"):
|
||||
if name.endswith(suffix):
|
||||
name = name[: -len(suffix)]
|
||||
return name
|
||||
|
||||
|
||||
def source_from_env(env: dict[str, str]) -> str | None:
|
||||
for key in ("src", "urls", "url", "cargoDeps", "npmDeps", "goModules"):
|
||||
val = env.get(key)
|
||||
if val and ("http" in val or "github" in val):
|
||||
return val
|
||||
for key, val in env.items():
|
||||
if key.lower().endswith("url") and val and ("http" in val or "github" in val):
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def infer_language(name: str, env: dict[str, str]) -> str | None:
|
||||
text = " ".join([name, env.get("nativeBuildInputs", ""), env.get("buildInputs", "")]).lower()
|
||||
if "python" in text or name.startswith("python"):
|
||||
return "Python"
|
||||
if "cargo" in text or "rustc" in text:
|
||||
return "Rust"
|
||||
if "go" in text and ("gomod" in text or "goModules" in env):
|
||||
return "Go"
|
||||
if "node" in text or "npm" in text or "pnpm" in text or "yarn" in text:
|
||||
return "JavaScript/TypeScript"
|
||||
if "cmake" in text or "gcc" in text or "clang" in text:
|
||||
return "C/C++"
|
||||
if name.startswith(("qt", "k", "lib")):
|
||||
return "C/C++"
|
||||
return None
|
||||
|
||||
|
||||
def static_upstream(name: str) -> dict[str, str] | None:
|
||||
base = re.sub(r"-\d+(?:\.\d+).*$", "", name)
|
||||
if base in COMMON_UPSTREAMS:
|
||||
source, language = COMMON_UPSTREAMS[base]
|
||||
return {"source_link": source, "language": language}
|
||||
if name.startswith("qt") or name in {"qca", "phonon", "poppler"}:
|
||||
return {"source_link": f"https://code.qt.io/cgit/qt/{base}.git", "language": "C++"}
|
||||
if name.startswith("gst-") or name == "gstreamer":
|
||||
project = "gstreamer" if name == "gstreamer" else base
|
||||
return {"source_link": f"https://gitlab.freedesktop.org/gstreamer/{project}", "language": "C"}
|
||||
kde_prefixes = (
|
||||
"karchive",
|
||||
"kauth",
|
||||
"kbookmarks",
|
||||
"kcmutils",
|
||||
"kcodecs",
|
||||
"kcompletion",
|
||||
"kconfig",
|
||||
"kconfigwidgets",
|
||||
"kcoreaddons",
|
||||
"kcrash",
|
||||
"kdbusaddons",
|
||||
"kdeclarative",
|
||||
"kded",
|
||||
"kdnssd",
|
||||
"kdoctools",
|
||||
"kfilemetadata",
|
||||
"kguiaddons",
|
||||
"ki18n",
|
||||
"kiconthemes",
|
||||
"kidletime",
|
||||
"kio",
|
||||
"kirigami",
|
||||
"kitemmodels",
|
||||
"kitemviews",
|
||||
"kjobwidgets",
|
||||
"knotifications",
|
||||
"kpackage",
|
||||
"kparts",
|
||||
"kpeople",
|
||||
"kpty",
|
||||
"kservice",
|
||||
"kstatusnotifieritem",
|
||||
"ksvg",
|
||||
"ktextwidgets",
|
||||
"kwallet",
|
||||
"kwidgetsaddons",
|
||||
"kwindowsystem",
|
||||
"kxmlgui",
|
||||
"solid",
|
||||
"sonnet",
|
||||
"syntax-highlighting",
|
||||
)
|
||||
if base.startswith(kde_prefixes):
|
||||
return {"source_link": f"https://invent.kde.org/frameworks/{base}", "language": "C++"}
|
||||
return None
|
||||
|
||||
|
||||
def github_repo(*values: str | None) -> str | None:
|
||||
for value in values:
|
||||
if not value:
|
||||
continue
|
||||
match = GITHUB_RE.search(value)
|
||||
if match:
|
||||
return f"{match.group('owner')}/{match.group('repo')}"
|
||||
return None
|
||||
|
||||
|
||||
def noisy_for_review(row: dict[str, Any]) -> bool:
|
||||
name = row["library"].lower()
|
||||
drv_path = row.get("drv_path", "").lower()
|
||||
if ".nupkg" in drv_path and not row.get("version_in_use"):
|
||||
return True
|
||||
noisy_exact = {
|
||||
"bash",
|
||||
"coreutils",
|
||||
"coreutils-full",
|
||||
"stdenv-linux",
|
||||
"install-shell-files",
|
||||
"version-check-hook",
|
||||
"writable-tmpdir-as-home-hook",
|
||||
"auto-patchelf-hook",
|
||||
"pkg-config-wrapper",
|
||||
"gcc-wrapper",
|
||||
"gnumake",
|
||||
"cmake",
|
||||
"ninja",
|
||||
"patchelf",
|
||||
"remove-references-to",
|
||||
"strip-nondeterminism",
|
||||
}
|
||||
if name in noisy_exact:
|
||||
return True
|
||||
noisy_bits = (
|
||||
"-source",
|
||||
"source-",
|
||||
"-go-modules",
|
||||
"builder.sh",
|
||||
"setup-hook",
|
||||
"-hook",
|
||||
".patch",
|
||||
".diff",
|
||||
"testdata",
|
||||
"fixture",
|
||||
)
|
||||
return any(bit in name for bit in noisy_bits)
|
||||
|
||||
|
||||
def github_json(path: str) -> dict[str, Any] | None:
|
||||
req = urllib.request.Request(
|
||||
f"https://api.github.com/{path}",
|
||||
headers={"Accept": "application/vnd.github+json", "User-Agent": "dotfiles-analysis"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as res:
|
||||
return json.loads(res.read().decode())
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError):
|
||||
return None
|
||||
|
||||
|
||||
def http_json(url: str) -> dict[str, Any] | None:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"Accept": "application/json", "User-Agent": "dotfiles-analysis"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as res:
|
||||
return json.loads(res.read().decode())
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def normalize_repo_url(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
value = value.strip()
|
||||
if value.startswith("git+"):
|
||||
value = value[4:]
|
||||
if value.startswith("git://github.com/"):
|
||||
value = "https://github.com/" + value.removeprefix("git://github.com/")
|
||||
if value.startswith("git@github.com:"):
|
||||
value = "https://github.com/" + value.removeprefix("git@github.com:")
|
||||
if value.endswith(".git"):
|
||||
value = value[:-4]
|
||||
return value
|
||||
|
||||
|
||||
def parse_ecosystem(row: dict[str, Any]) -> tuple[str | None, str | None, str | None]:
|
||||
name = row["library"]
|
||||
version = row.get("version_in_use") or None
|
||||
drv = row.get("drv_path", "")
|
||||
is_nuget_like = name.startswith(NUGET_NAME_PREFIXES)
|
||||
if ".nupkg" in drv or is_nuget_like:
|
||||
# The derivation rows have clean name/version; the raw .nupkg rows are
|
||||
# filtered from review but can still be enriched in summary/deps.
|
||||
if not version and ".nupkg" in drv:
|
||||
base = clean_library_name(Path(drv).name.removesuffix(".drv"))
|
||||
m = re.match(r"(.+)\.(\d+(?:\.\d+)+(?:[-.][0-9A-Za-z]+)*)$", base)
|
||||
if m:
|
||||
name, version = m.group(1), m.group(2)
|
||||
return "nuget", name, version
|
||||
if row["root_name"] in {"nix-serve"} and "perl5." in drv:
|
||||
return "cpan", name, version
|
||||
if name.startswith("python") or "python" in drv:
|
||||
py_name = re.sub(r"^python\d+(?:\.\d+)?-", "", name)
|
||||
return "pypi", py_name, version
|
||||
if "node_modules" in row.get("dependency_path", "") or row["root_name"] in {"karakeep", "uptime-kuma"}:
|
||||
return "npm", name, version
|
||||
if "cargo" in name.lower() or "rust" in drv.lower():
|
||||
return "crates", name, version
|
||||
return None, None, None
|
||||
|
||||
|
||||
def apply_ecosystem_overrides(ecosystem: str, package: str, result: dict[str, Any]) -> dict[str, Any]:
|
||||
if ecosystem == "nuget":
|
||||
source = NUGET_REPO_OVERRIDES.get(package)
|
||||
if not source and package.startswith("Microsoft.AspNetCore."):
|
||||
source = "https://github.com/dotnet/aspnetcore"
|
||||
if not source and package.startswith("Microsoft.EntityFrameworkCore"):
|
||||
source = "https://github.com/dotnet/efcore"
|
||||
if not source and package.startswith("Microsoft.Build"):
|
||||
source = "https://github.com/dotnet/msbuild"
|
||||
if not source and package.startswith("Microsoft.Identity.Client"):
|
||||
source = "https://github.com/AzureAD/microsoft-authentication-library-for-dotnet"
|
||||
if not source and (package.startswith("Microsoft.") or package.startswith("System.") or package.startswith("runtime.")):
|
||||
source = "https://github.com/dotnet/runtime"
|
||||
if source:
|
||||
result["source_link"] = source
|
||||
result["github_repo"] = github_repo(source)
|
||||
result.setdefault("language", "C#")
|
||||
return result
|
||||
|
||||
|
||||
def release_date_from_pypi(files: list[dict[str, Any]]) -> str | None:
|
||||
dates = [f.get("upload_time_iso_8601") for f in files if f.get("upload_time_iso_8601")]
|
||||
return min(dates) if dates else None
|
||||
|
||||
|
||||
def enrich_ecosystem(ecosystem: str, package: str, version: str | None, cache: dict[str, Any]) -> dict[str, Any]:
|
||||
key = ecosystem_cache_key(ecosystem, package, version)
|
||||
if key in cache:
|
||||
return cache[key]
|
||||
result: dict[str, Any] = {"ecosystem": ecosystem}
|
||||
quoted = urllib.parse.quote(package, safe="")
|
||||
|
||||
if ecosystem == "nuget":
|
||||
result["language"] = "C#"
|
||||
if version:
|
||||
data = http_json(f"https://api.nuget.org/v3/registration5-semver1/{package.lower()}/{version.lower()}.json")
|
||||
entry = (data or {}).get("catalogEntry", {})
|
||||
if isinstance(entry, str):
|
||||
entry = http_json(entry) or {}
|
||||
repo = entry.get("repository") or {}
|
||||
repo_url = repo.get("url") if isinstance(repo, dict) else None
|
||||
repo_url = normalize_repo_url(repo_url or entry.get("repositoryUrl") or entry.get("projectUrl"))
|
||||
result.update(
|
||||
{
|
||||
"source_link": repo_url or entry.get("projectUrl"),
|
||||
"release_date": entry.get("published"),
|
||||
}
|
||||
)
|
||||
index = http_json(f"https://api.nuget.org/v3-flatcontainer/{package.lower()}/index.json")
|
||||
versions = (index or {}).get("versions") or []
|
||||
if versions:
|
||||
result["latest_version"] = versions[-1]
|
||||
|
||||
elif ecosystem == "npm":
|
||||
data = http_json(f"https://registry.npmjs.org/{quoted}") or {}
|
||||
info = data.get("versions", {}).get(version or "", {}) if version else {}
|
||||
repo = info.get("repository") or data.get("repository") or {}
|
||||
repo_url = repo.get("url") if isinstance(repo, dict) else repo
|
||||
latest = (data.get("dist-tags") or {}).get("latest")
|
||||
result.update(
|
||||
{
|
||||
"source_link": normalize_repo_url(repo_url) or data.get("homepage"),
|
||||
"latest_version": latest,
|
||||
"release_date": (data.get("time") or {}).get(version or ""),
|
||||
"latest_release_date": (data.get("time") or {}).get(latest or ""),
|
||||
"language": "JavaScript/TypeScript",
|
||||
}
|
||||
)
|
||||
|
||||
elif ecosystem == "pypi":
|
||||
data = http_json(f"https://pypi.org/pypi/{quoted}/json") or {}
|
||||
info = data.get("info", {})
|
||||
urls = info.get("project_urls") or {}
|
||||
source = urls.get("Source") or urls.get("Source Code") or urls.get("Homepage") or info.get("home_page") or info.get("package_url")
|
||||
latest = info.get("version")
|
||||
result.update(
|
||||
{
|
||||
"source_link": normalize_repo_url(source),
|
||||
"latest_version": latest,
|
||||
"release_date": release_date_from_pypi((data.get("releases") or {}).get(version or "", [])),
|
||||
"latest_release_date": release_date_from_pypi((data.get("releases") or {}).get(latest or "", [])),
|
||||
"language": "Python",
|
||||
}
|
||||
)
|
||||
|
||||
elif ecosystem == "crates":
|
||||
data = http_json(f"https://crates.io/api/v1/crates/{quoted}") or {}
|
||||
crate = data.get("crate", {})
|
||||
result.update(
|
||||
{
|
||||
"source_link": normalize_repo_url(crate.get("repository") or crate.get("homepage")),
|
||||
"latest_version": crate.get("max_stable_version") or crate.get("newest_version"),
|
||||
"latest_release_date": crate.get("updated_at"),
|
||||
"language": "Rust",
|
||||
}
|
||||
)
|
||||
|
||||
elif ecosystem == "cpan":
|
||||
dist = package.replace("::", "-")
|
||||
result.update(
|
||||
{
|
||||
"source_link": f"https://metacpan.org/pod/{package}",
|
||||
"language": "Perl",
|
||||
}
|
||||
)
|
||||
data = http_json(f"https://fastapi.metacpan.org/v1/release/{urllib.parse.quote(dist, safe='')}") or {}
|
||||
resources = ((data.get("metadata") or {}).get("resources") or {})
|
||||
repo = resources.get("repository") or {}
|
||||
repo_url = repo.get("url") if isinstance(repo, dict) else repo
|
||||
result.update(
|
||||
{
|
||||
"source_link": normalize_repo_url(repo_url) or result["source_link"],
|
||||
"latest_version": data.get("version"),
|
||||
"latest_release_date": data.get("date"),
|
||||
}
|
||||
)
|
||||
if str(data.get("version")) == str(version):
|
||||
result["release_date"] = data.get("date")
|
||||
|
||||
result["github_repo"] = github_repo(result.get("source_link"))
|
||||
result = apply_ecosystem_overrides(ecosystem, package, result)
|
||||
cache[key] = result
|
||||
return result
|
||||
|
||||
|
||||
def ecosystem_cache_key(ecosystem: str | None, package: str | None, version: str | None) -> str:
|
||||
return f"{ecosystem}:{package}:{version or ''}"
|
||||
|
||||
|
||||
def enrich_github(repo: str, cache: dict[str, Any], sleep: float) -> dict[str, Any]:
|
||||
if repo in cache:
|
||||
return cache[repo]
|
||||
data = github_json(f"repos/{repo}") or {}
|
||||
if sleep:
|
||||
time.sleep(sleep)
|
||||
latest = github_json(f"repos/{repo}/releases/latest") or {}
|
||||
if sleep:
|
||||
time.sleep(sleep)
|
||||
result = {
|
||||
"github_repo": repo,
|
||||
"github_stars": data.get("stargazers_count"),
|
||||
"language": data.get("language"),
|
||||
"source_link": data.get("html_url"),
|
||||
"latest_version": latest.get("tag_name"),
|
||||
"latest_release_date": latest.get("published_at"),
|
||||
}
|
||||
cache[repo] = result
|
||||
return result
|
||||
|
||||
|
||||
def walk_deps(root: dict[str, Any], all_drvs: dict[str, Any], max_depth: int) -> list[dict[str, Any]]:
|
||||
start = root.get("drv")
|
||||
if not start:
|
||||
return []
|
||||
start_key = Path(start).name
|
||||
rows = []
|
||||
seen = {start_key}
|
||||
queue = deque([(start_key, [], 0)])
|
||||
while queue:
|
||||
drv, path, depth = queue.popleft()
|
||||
if depth >= max_depth:
|
||||
continue
|
||||
item = all_drvs.get(drv, {})
|
||||
input_drvs = item.get("inputDrvs") or (item.get("inputs") or {}).get("drvs") or {}
|
||||
for dep_drv in sorted(input_drvs.keys()):
|
||||
dep_key = Path(dep_drv).name
|
||||
if dep_key in seen:
|
||||
continue
|
||||
seen.add(dep_key)
|
||||
meta = drv_meta(dep_key, all_drvs)
|
||||
static = static_upstream(meta["name"]) or {}
|
||||
source_link = meta["source_link"] or static.get("source_link")
|
||||
language = meta["language"] or static.get("language")
|
||||
dep_path = path + [meta["name"]]
|
||||
rows.append(
|
||||
{
|
||||
"host": root["host"],
|
||||
"root_kind": root["kind"],
|
||||
"root_name": root["rootName"],
|
||||
"root_package": root["packageName"],
|
||||
"library": meta["name"],
|
||||
"version_in_use": meta["version"],
|
||||
"dep_depth": depth + 1,
|
||||
"dependency_path": " -> ".join([root["rootName"]] + dep_path),
|
||||
"drv_path": dep_key,
|
||||
"homepage": meta["homepage"],
|
||||
"source_link": source_link,
|
||||
"language": language,
|
||||
"github_repo": github_repo(meta["homepage"], source_link),
|
||||
"github_stars": None,
|
||||
"ecosystem": None,
|
||||
"release_date": None,
|
||||
"latest_version": None,
|
||||
"latest_release_date": None,
|
||||
}
|
||||
)
|
||||
queue.append((dep_key, dep_path, depth + 1))
|
||||
return rows
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]], fields: list[str]) -> None:
|
||||
with path.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def ecosystem_priority(ecosystem: str | None) -> int:
|
||||
return {
|
||||
"cpan": 0,
|
||||
"npm": 1,
|
||||
"pypi": 2,
|
||||
"crates": 3,
|
||||
"nuget": 4,
|
||||
}.get(ecosystem or "", 9)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-roots", type=int, default=18)
|
||||
parser.add_argument("--max-depth", type=int, default=2)
|
||||
parser.add_argument("--github-limit", type=int, default=80)
|
||||
parser.add_argument("--github-sleep", type=float, default=0.1)
|
||||
parser.add_argument("--ecosystem-limit", type=int, default=400)
|
||||
args = parser.parse_args()
|
||||
|
||||
OUT.mkdir(exist_ok=True)
|
||||
roots = eval_roots()
|
||||
selected = [r for r in roots if r.get("drv")][: args.max_roots]
|
||||
|
||||
dep_rows: list[dict[str, Any]] = []
|
||||
for root in selected:
|
||||
print(f"walking {root['host']}:{root['rootName']} {root['packageName']}", file=sys.stderr)
|
||||
try:
|
||||
all_drvs = derivation_show_recursive(root["drv"])
|
||||
except RuntimeError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
continue
|
||||
dep_rows.extend(walk_deps(root, all_drvs, args.max_depth))
|
||||
|
||||
ecosystem_cache_path = OUT / "ecosystem-cache.json"
|
||||
ecosystem_cache = json.loads(ecosystem_cache_path.read_text()) if ecosystem_cache_path.exists() else {}
|
||||
ecosystem_keys = []
|
||||
ecosystem_key_scores: dict[tuple[str | None, str | None, str | None], tuple[int, int, str, str]] = {}
|
||||
ecosystem_rows: dict[tuple[str | None, str | None, str | None], list[dict[str, Any]]] = {}
|
||||
for row in dep_rows:
|
||||
ecosystem, package, version = parse_ecosystem(row)
|
||||
if ecosystem and package:
|
||||
key = (ecosystem, package, version)
|
||||
ecosystem_rows.setdefault(key, []).append(row)
|
||||
if key not in ecosystem_keys:
|
||||
ecosystem_keys.append(key)
|
||||
review_score = 0 if not noisy_for_review(row) else 1
|
||||
score = (review_score, ecosystem_priority(ecosystem), package.lower(), version or "")
|
||||
if key not in ecosystem_key_scores or score < ecosystem_key_scores[key]:
|
||||
ecosystem_key_scores[key] = score
|
||||
ecosystem_keys.sort(key=lambda key: ecosystem_key_scores.get(key, (9, 9, "", "")))
|
||||
selected_ecosystem_keys = ecosystem_keys if args.ecosystem_limit < 0 else ecosystem_keys[: args.ecosystem_limit]
|
||||
for idx, (ecosystem, package, version) in enumerate(selected_ecosystem_keys, start=1):
|
||||
if idx % 25 == 1:
|
||||
print(f"enriching ecosystem metadata {idx}/{len(selected_ecosystem_keys)}", file=sys.stderr)
|
||||
meta = enrich_ecosystem(ecosystem, package, version, ecosystem_cache)
|
||||
for row in ecosystem_rows.get((ecosystem, package, version), []):
|
||||
row.update({k: v for k, v in meta.items() if v is not None and (not row.get(k) or k in {"ecosystem", "release_date"})})
|
||||
if idx % 25 == 0:
|
||||
write_json_atomic(ecosystem_cache_path, ecosystem_cache)
|
||||
selected_ecosystem_key_set = set(selected_ecosystem_keys)
|
||||
cached_only_keys = [
|
||||
key
|
||||
for key in ecosystem_keys
|
||||
if ecosystem_cache_key(*key) in ecosystem_cache and key not in selected_ecosystem_key_set
|
||||
]
|
||||
for ecosystem, package, version in cached_only_keys:
|
||||
meta = ecosystem_cache[ecosystem_cache_key(ecosystem, package, version)]
|
||||
for row in ecosystem_rows.get((ecosystem, package, version), []):
|
||||
row.update({k: v for k, v in meta.items() if v is not None and (not row.get(k) or k in {"ecosystem", "release_date"})})
|
||||
write_json_atomic(ecosystem_cache_path, ecosystem_cache)
|
||||
|
||||
cache_path = OUT / "github-cache.json"
|
||||
cache = json.loads(cache_path.read_text()) if cache_path.exists() else {}
|
||||
repos = []
|
||||
for row in dep_rows:
|
||||
repo = row.get("github_repo")
|
||||
if repo and repo not in repos:
|
||||
repos.append(repo)
|
||||
for idx, repo in enumerate(repos[: args.github_limit], start=1):
|
||||
if idx % 25 == 1:
|
||||
print(f"enriching GitHub metadata {idx}/{min(len(repos), args.github_limit)}", file=sys.stderr)
|
||||
gh = enrich_github(repo, cache, args.github_sleep)
|
||||
for row in dep_rows:
|
||||
if row.get("github_repo") == repo:
|
||||
row.update({k: v for k, v in gh.items() if v is not None})
|
||||
if idx % 25 == 0:
|
||||
write_json_atomic(cache_path, cache)
|
||||
|
||||
for root in roots:
|
||||
repo = github_repo(root.get("homepage"), " ".join(root.get("sourceUrls") or []))
|
||||
root["github_repo"] = repo
|
||||
root["github_stars"] = None
|
||||
root["ecosystem"] = "nix"
|
||||
root["release_date"] = None
|
||||
root["latest_version"] = None
|
||||
root["latest_release_date"] = None
|
||||
root["language"] = None
|
||||
if repo:
|
||||
gh = enrich_github(repo, cache, args.github_sleep)
|
||||
root.update({k: v for k, v in gh.items() if v is not None})
|
||||
write_json_atomic(cache_path, cache)
|
||||
|
||||
root_fields = [
|
||||
"priority",
|
||||
"host",
|
||||
"kind",
|
||||
"rootName",
|
||||
"packageName",
|
||||
"pname",
|
||||
"version",
|
||||
"drv",
|
||||
"storePath",
|
||||
"homepage",
|
||||
"description",
|
||||
"sourceUrls",
|
||||
"image",
|
||||
"github_repo",
|
||||
"github_stars",
|
||||
"ecosystem",
|
||||
"release_date",
|
||||
"latest_version",
|
||||
"latest_release_date",
|
||||
"language",
|
||||
]
|
||||
dep_fields = [
|
||||
"host",
|
||||
"root_kind",
|
||||
"root_name",
|
||||
"root_package",
|
||||
"library",
|
||||
"version_in_use",
|
||||
"dep_depth",
|
||||
"dependency_path",
|
||||
"drv_path",
|
||||
"homepage",
|
||||
"source_link",
|
||||
"github_repo",
|
||||
"github_stars",
|
||||
"ecosystem",
|
||||
"release_date",
|
||||
"latest_version",
|
||||
"latest_release_date",
|
||||
"language",
|
||||
]
|
||||
write_csv(OUT / "network-package-roots.csv", roots, root_fields)
|
||||
write_csv(OUT / "network-library-deps.csv", dep_rows, dep_fields)
|
||||
|
||||
# One row per library, preserving the first root/path encountered. This is
|
||||
# convenient for hand-reviewing uncommon deps before opening the full edge CSV.
|
||||
summary: dict[str, dict[str, Any]] = {}
|
||||
for row in dep_rows:
|
||||
key = row["drv_path"]
|
||||
summary.setdefault(key, row.copy())
|
||||
write_csv(
|
||||
OUT / "network-library-summary.csv",
|
||||
sorted(summary.values(), key=lambda r: (r.get("github_stars") is not None, r.get("github_stars") or 0, r["library"])),
|
||||
dep_fields,
|
||||
)
|
||||
review_rows = [r for r in summary.values() if not noisy_for_review(r)]
|
||||
write_csv(
|
||||
OUT / "network-library-review.csv",
|
||||
sorted(review_rows, key=lambda r: (r.get("github_stars") is not None, r.get("github_stars") or 0, r["library"])),
|
||||
dep_fields,
|
||||
)
|
||||
print(f"wrote {len(roots)} roots and {len(dep_rows)} dependency rows", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
222
analysis/low-star-network-path-packages.md
Normal file
222
analysis/low-star-network-path-packages.md
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
# Low-Star Packages On Plausible Network/Data Paths
|
||||
|
||||
Generated from `analysis/network-library-review.csv` and GitHub metadata on 2026-05-30. Star counts are GitHub stars at collection time. Dependency paths are Nix derivation/package paths, so they show that a package is reachable from the configured service package closure; they do not prove every library is loaded on every runtime request path.
|
||||
|
||||
Selection criteria: GitHub-backed dependency with relatively low stars, used by a network-facing root (`nix-serve`, `prowlarr`, `jellyfin`, `sonarr`, `radarr`), and plausibly involved in HTTP parsing, socket handling, JSON/XML/HTML parsing, remote metadata parsing, text normalization, database access, or similar externally influenced data handling.
|
||||
|
||||
## Highest Priority
|
||||
|
||||
| Project | Stars | Used by | Version in use | Latest seen | Why it may matter |
|
||||
| --- | ---: | --- | --- | --- | --- |
|
||||
| [kazeburo/HTTP-Entity-Parser](https://github.com/kazeburo/HTTP-Entity-Parser) | 5 | `nix-serve` | `0.25` | `0.25` | PSGI-compliant HTTP entity/body parser, directly adjacent to HTTP request handling. |
|
||||
| [kazuho/p5-http-parser-xs](https://github.com/kazuho/p5-http-parser-xs) | 30 | `nix-serve` | `0.17` | `0.17` | Fast C/XS HTTP parser used through the Perl web stack; low-level parser code is a high-value review target. |
|
||||
| [shlomif/perl-io-socket-inet6](https://github.com/shlomif/perl-io-socket-inet6) | 0 | `nix-serve` | `2.73` | `2.73` | IPv6 socket support library in the `nix-serve` Perl closure. Socket plumbing is network-path relevant. |
|
||||
| [AngleSharp/AngleSharp.Xml](https://github.com/AngleSharp/AngleSharp.Xml) | 20 | `prowlarr` | `1.0.0` | `1.0.0` | XML and DTD parser extension for AngleSharp. Prowlarr handles indexer feeds/pages from remote sources. |
|
||||
| [p5sagit/JSON-MaybeXS](https://github.com/p5sagit/JSON-MaybeXS) | 4 | `nix-serve` | `1.004005` | `1.004008` | JSON backend selection/compatibility module in the HTTP service closure. JSON parsing often receives externally supplied data. |
|
||||
|
||||
## Medium Priority
|
||||
|
||||
| Project | Stars | Used by | Version in use | Latest seen | Notes |
|
||||
| --- | ---: | --- | --- | --- | --- |
|
||||
| [madsen/io-html](https://github.com/madsen/io-html) | 3 | `nix-serve` | `1.004` | `1.004` | Perl module for opening files with automatic charset detection. Less directly exposed than HTTP parsers, but charset detection can be input-sensitive. |
|
||||
| [Zastai/MetaBrainz.MusicBrainz](https://github.com/Zastai/MetaBrainz.MusicBrainz) | 41 | `jellyfin` | `6.1.0` | `v8.0.1` | Native .NET implementation of MusicBrainz client/data model. Jellyfin can ingest remote metadata responses. |
|
||||
| [Zastai/MetaBrainz.Common.Json](https://github.com/Zastai/MetaBrainz.Common.Json) | 1 | `jellyfin` | `6.0.2` | `v7.2.0` | JSON helper classes for MetaBrainz packages. Relevant to parsing remote metadata. |
|
||||
| [Zastai/MetaBrainz.Common](https://github.com/Zastai/MetaBrainz.Common) | 0 | `jellyfin` | `3.0.0` | `v4.1.1` | Shared classes for MetaBrainz packages. Low stars and in the metadata path, but not itself a parser entry point. |
|
||||
| [NightOwl888/ICU4N](https://github.com/NightOwl888/ICU4N) | 44 | `jellyfin` | `60.1.0-alpha.356` | `60.1.0-alpha.439` | Unicode/text normalization and transliteration library. Useful to review because media metadata and filenames are attacker-influenced in many deployments. |
|
||||
|
||||
## Lower Priority But Network-Adjacent
|
||||
|
||||
| Project | Stars | Used by | Version in use | Latest seen | Notes |
|
||||
| --- | ---: | --- | --- | --- | --- |
|
||||
| [ericsink/SQLitePCL.raw](https://github.com/ericsink/SQLitePCL.raw) | 609 | `jellyfin` | `2.1.10` | `v3.0.3` | Low-level SQLite access layer. Not a network parser, but stores/query data derived from remote/user-controlled metadata. |
|
||||
| [dotnet/SqlClient](https://github.com/dotnet/SqlClient) | 974 | `sonarr`, `radarr` | `2.1.7`, `6.1.1`, SNI runtime `2.1.1`, `6.0.2` | `v7.0.1` | SQL Server connectivity. Relevant if these apps are configured to use SQL Server or process DB connection data, but less relevant for the default SQLite-style local deployment path. |
|
||||
|
||||
## Candidate Details
|
||||
|
||||
### kazeburo/HTTP-Entity-Parser
|
||||
|
||||
Project: [https://github.com/kazeburo/HTTP-Entity-Parser](https://github.com/kazeburo/HTTP-Entity-Parser)
|
||||
|
||||
Description: PSGI compliant HTTP Entity Parser.
|
||||
|
||||
Used by: `nix-serve`
|
||||
|
||||
Dependency path: `nix-serve -> perl-5.42.0-env -> HTTP-Entity-Parser`
|
||||
|
||||
Version in use: `0.25`
|
||||
|
||||
Latest/release data: latest `0.25`, latest release date `2020-11-28T02:35:43`
|
||||
|
||||
Other data: Perl, 5 stars, 8 forks, 2 open issues, not archived, last pushed `2020-11-28T02:35:43Z`, license `NOASSERTION`
|
||||
|
||||
Assessment: Directly relevant to HTTP body parsing for `nix-serve`; worth manual review if `nix-serve` is publicly exposed through Caddy.
|
||||
|
||||
### kazuho/p5-http-parser-xs
|
||||
|
||||
Project: [https://github.com/kazuho/p5-http-parser-xs](https://github.com/kazuho/p5-http-parser-xs)
|
||||
|
||||
Description: Fast HTTP parser.
|
||||
|
||||
Used by: `nix-serve`
|
||||
|
||||
Dependency path: `nix-serve -> perl-5.42.0-env -> HTTP-Parser-XS`
|
||||
|
||||
Version in use: `0.17`
|
||||
|
||||
Latest/release data: latest `0.17`, latest release date `2014-12-15T07:53:06`
|
||||
|
||||
Other data: C, 30 stars, 11 forks, 9 open issues, not archived, last pushed `2024-06-13T04:08:54Z`
|
||||
|
||||
Assessment: Highest-value low-star item because it is C parser code close to HTTP request parsing.
|
||||
|
||||
### shlomif/perl-io-socket-inet6
|
||||
|
||||
Project: [https://github.com/shlomif/perl-io-socket-inet6](https://github.com/shlomif/perl-io-socket-inet6)
|
||||
|
||||
Description: CPAN IPv6 socket module mirror/repository.
|
||||
|
||||
Used by: `nix-serve`
|
||||
|
||||
Dependency path: `nix-serve -> perl-5.42.0-env -> IO-Socket-INET6`
|
||||
|
||||
Version in use: `2.73`
|
||||
|
||||
Latest/release data: latest `2.73`, latest release date `2021-12-10T07:31:35`
|
||||
|
||||
Other data: Perl, 0 stars, 1 fork, 0 open issues, not archived, last pushed `2021-12-10T07:31:26Z`, license `NOASSERTION`
|
||||
|
||||
Assessment: Network plumbing dependency. Lower parser risk than HTTP parsers, but the star count is effectively zero.
|
||||
|
||||
### AngleSharp/AngleSharp.Xml
|
||||
|
||||
Project: [https://github.com/AngleSharp/AngleSharp.Xml](https://github.com/AngleSharp/AngleSharp.Xml)
|
||||
|
||||
Description: Library adding XML and DTD parsing capabilities to AngleSharp.
|
||||
|
||||
Used by: `prowlarr`
|
||||
|
||||
Dependency path: `prowlarr -> AngleSharp.Xml`
|
||||
|
||||
Version in use: `1.0.0`
|
||||
|
||||
Latest/release data: latest `1.0.0`, release date `2023-01-15T12:45:03.84Z`, latest release date `2023-01-15T12:45:04Z`
|
||||
|
||||
Other data: C#, 20 stars, 6 forks, 5 open issues, not archived, last pushed `2025-01-26T20:54:26Z`, license `MIT`
|
||||
|
||||
Assessment: XML/DTD parsing in an indexer-facing service is plausibly exposed to remote feed/page content. Worth checking DTD/external entity behavior and parser limits.
|
||||
|
||||
### p5sagit/JSON-MaybeXS
|
||||
|
||||
Project: [https://github.com/p5sagit/JSON-MaybeXS](https://github.com/p5sagit/JSON-MaybeXS)
|
||||
|
||||
Description: JSON backend compatibility/selecting module for Perl.
|
||||
|
||||
Used by: `nix-serve`
|
||||
|
||||
Dependency path: `nix-serve -> perl-5.42.0-env -> JSON-MaybeXS`
|
||||
|
||||
Version in use: `1.004005`
|
||||
|
||||
Latest/release data: latest `1.004008`, latest release date `2024-08-10T20:23:23`
|
||||
|
||||
Other data: Perl, 4 stars, 6 forks, 1 open issue, not archived, last pushed `2024-12-27T11:55:18Z`
|
||||
|
||||
Assessment: Probably a wrapper rather than the parser implementation itself, but it is in a web service closure and touches JSON handling.
|
||||
|
||||
### madsen/io-html
|
||||
|
||||
Project: [https://github.com/madsen/io-html](https://github.com/madsen/io-html)
|
||||
|
||||
Description: Perl module that opens a file and performs automatic charset detection.
|
||||
|
||||
Used by: `nix-serve`
|
||||
|
||||
Dependency path: `nix-serve -> perl-5.42.0-env -> IO-HTML`
|
||||
|
||||
Version in use: `1.004`
|
||||
|
||||
Latest/release data: latest `1.004`, latest release date `2020-09-26T16:52:29`
|
||||
|
||||
Other data: Perl, 3 stars, 1 fork, 0 open issues, not archived, last pushed `2020-09-26T16:51:31Z`
|
||||
|
||||
Assessment: Charset detection can be input-sensitive, but this is lower priority unless `nix-serve` uses it on request-supplied content.
|
||||
|
||||
### Zastai MetaBrainz packages
|
||||
|
||||
Projects: [MetaBrainz.Common](https://github.com/Zastai/MetaBrainz.Common), [MetaBrainz.Common.Json](https://github.com/Zastai/MetaBrainz.Common.Json), [MetaBrainz.MusicBrainz](https://github.com/Zastai/MetaBrainz.MusicBrainz)
|
||||
|
||||
Descriptions: Shared classes, JSON helpers, and native .NET implementation of libmusicbrainz.
|
||||
|
||||
Used by: `jellyfin`
|
||||
|
||||
Dependency paths: `jellyfin -> MetaBrainz.Common`, `jellyfin -> MetaBrainz.Common.Json`, `jellyfin -> MetaBrainz.MusicBrainz`
|
||||
|
||||
Versions in use: `3.0.0`, `6.0.2`, `6.1.0`
|
||||
|
||||
Latest/release data: latest `v4.1.1`, `v7.2.0`, `v8.0.1`; latest release dates in 2026 for all three
|
||||
|
||||
Other data: C#, 0/1/41 stars, 0/0/10 forks, not archived, MIT license
|
||||
|
||||
Assessment: These are in Jellyfin metadata handling. They are not direct socket parsers, but they process metadata structures that can originate from remote services or media tags.
|
||||
|
||||
### NightOwl888/ICU4N
|
||||
|
||||
Project: [https://github.com/NightOwl888/ICU4N](https://github.com/NightOwl888/ICU4N)
|
||||
|
||||
Description: International Components for Unicode for .NET.
|
||||
|
||||
Used by: `jellyfin`
|
||||
|
||||
Dependency paths: `jellyfin -> ICU4N`, `jellyfin -> ICU4N.Transliterator`
|
||||
|
||||
Version in use: `60.1.0-alpha.356`
|
||||
|
||||
Latest/release data: latest `60.1.0-alpha.439` for `ICU4N`; latest `60.1.0-alpha.356` for `ICU4N.Transliterator`; NuGet release dates were not exposed in the cached data
|
||||
|
||||
Other data: C#, 44 stars, 8 forks, 22 open issues, not archived, last pushed `2026-05-08T23:25:53Z`, license `Apache-2.0`
|
||||
|
||||
Assessment: Text normalization/transliteration libraries can receive untrusted metadata, filenames, subtitles, and tags. Alpha-version package in use is notable.
|
||||
|
||||
### ericsink/SQLitePCL.raw
|
||||
|
||||
Project: [https://github.com/ericsink/SQLitePCL.raw](https://github.com/ericsink/SQLitePCL.raw)
|
||||
|
||||
Description: Portable Class Library for low-level raw access to SQLite.
|
||||
|
||||
Used by: `jellyfin`
|
||||
|
||||
Dependency paths: `jellyfin -> SQLitePCLRaw.core`, `jellyfin -> SQLitePCLRaw.bundle_e_sqlite3`, `jellyfin -> SQLitePCLRaw.lib.e_sqlite3`, `jellyfin -> SQLitePCLRaw.provider.e_sqlite3`
|
||||
|
||||
Version in use: `2.1.10`
|
||||
|
||||
Latest/release data: latest `v3.0.3`, release dates around `2024-09-11`, latest release date `2026-05-07T17:28:57Z`
|
||||
|
||||
Other data: C#, 609 stars, 134 forks, 36 open issues, not archived, last pushed `2026-05-07T17:23:42Z`, license `Apache-2.0`
|
||||
|
||||
Assessment: Not a network parser, but stores and queries data derived from network/media metadata. Lower priority than parser/socket libraries.
|
||||
|
||||
### dotnet/SqlClient
|
||||
|
||||
Project: [https://github.com/dotnet/SqlClient](https://github.com/dotnet/SqlClient)
|
||||
|
||||
Description: Microsoft.Data.SqlClient provides database connectivity to SQL Server for .NET applications.
|
||||
|
||||
Used by: `sonarr`, `radarr`
|
||||
|
||||
Dependency paths: `sonarr -> Microsoft.Data.SqlClient`, `radarr -> Microsoft.Data.SqlClient`, and corresponding `Microsoft.Data.SqlClient.SNI.runtime` rows
|
||||
|
||||
Versions in use: `2.1.7`, `6.1.1`, SNI runtime `2.1.1`, `6.0.2`
|
||||
|
||||
Latest/release data: latest `v7.0.1`, latest release date `2026-04-24T19:34:24Z`
|
||||
|
||||
Other data: C#, 974 stars, 330 forks, 276 open issues, not archived, last pushed `2026-05-30T11:30:25Z`, license `MIT`
|
||||
|
||||
Assessment: Network-adjacent database client. Relevant mainly if Sonarr/Radarr are configured to use SQL Server or expose database connection handling.
|
||||
|
||||
## Low-Star Items Not Prioritized
|
||||
|
||||
These appeared in the low-star scan but are less plausibly on a network/data parsing path: [garu/data-dump](https://github.com/garu/data-dump), [garu/Clone](https://github.com/garu/Clone), Serilog extension/sink packages, NUnit test adapters, and `buildcatrust`. They may still matter for build integrity or diagnostics, but they are not obvious request/response parser or socket-facing dependencies from the current dependency paths.
|
||||
|
||||
## Suggested Follow-Up
|
||||
|
||||
Review `nix-serve` first because it is exposed through Caddy and has several very low-star Perl HTTP/socket parser dependencies. Then check `prowlarr` XML/HTML parsing behavior, especially external entity handling and parser size/time limits. Finally, decide whether Jellyfin remote metadata providers are enabled and exposed enough to justify deeper review of the MetaBrainz and ICU4N paths.
|
||||
207
analysis/nix-serve-security-entry-points.md
Normal file
207
analysis/nix-serve-security-entry-points.md
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# nix-serve security entry points
|
||||
|
||||
Target deployment:
|
||||
|
||||
- Local: `nix-serve` / Starman directly on `:5000`.
|
||||
- Public: `nix.fern.danbulant.cloud:80` via Caddy `reverse_proxy http://localhost:${config.services.nix-serve.port}` in `servers/fern/configuration.nix`.
|
||||
|
||||
Primary application code:
|
||||
|
||||
- `analysis/nix-serve/nix-serve.psgi`
|
||||
- `analysis/p5-http-parser-xs/XS.xs`
|
||||
- `analysis/p5-http-parser-xs/picohttpparser/picohttpparser.c`
|
||||
- `analysis/HTTP-Entity-Parser/lib/HTTP/Entity/Parser*.pm`
|
||||
|
||||
## Request Flow
|
||||
|
||||
External request reaches Caddy, then Starman, then the PSGI app. Starman uses `HTTP::Parser::XS` to parse the request line and headers into PSGI env values. `nix-serve.psgi` routes only on `$env->{PATH_INFO}` and ignores method, query string, and request body.
|
||||
|
||||
Application routes in `nix-serve.psgi`:
|
||||
|
||||
- `/nix-cache-info`: static cache metadata.
|
||||
- `/<hash>.narinfo`: maps hash prefix to a store path and returns NAR metadata/signatures.
|
||||
- `/nar/<hash>-<narhash>.nar`: maps hash prefix, checks NAR hash, then spawns `nix store dump-path -- <storePath>`.
|
||||
- `/nar/<hash>.nar`: legacy endpoint, maps hash prefix, then spawns `nix store dump-path -- <storePath>` without the NAR hash check.
|
||||
- `/log/<hash>-<name>`: constructs `/nix/store/<hash>-<name>` and spawns `nix log <storePath>` without first proving that the path is valid or present.
|
||||
|
||||
## Confirmed Active Behaviors
|
||||
|
||||
### A. Incomplete `Content-Length` kills a Starman worker
|
||||
|
||||
Starman reads request bodies before dispatching to `nix-serve.psgi`, even though the app ignores bodies. In `Starman/Server.pm:450-462`, a positive `CONTENT_LENGTH` causes `_prepare_env` to read until the declared length is consumed. If the client closes early, it executes `die "Read error: $!\n"` outside an eval around request processing.
|
||||
|
||||
Confirmed behavior:
|
||||
|
||||
- Direct `:5000`: sending `Content-Length: 999999` with a one-byte body and closing replaces one worker process.
|
||||
- Through Caddy: the same incomplete request to `nix.fern.danbulant.cloud:80` also replaces one Starman worker.
|
||||
- The master process respawns the worker, so this is a repeatable worker-crash / availability issue rather than a one-shot full service crash.
|
||||
|
||||
Observed worker replacement example:
|
||||
|
||||
```text
|
||||
before: 2529 2530 2532 2533 1239489
|
||||
after: 2530 2532 2533 1239489 1240067
|
||||
```
|
||||
|
||||
Root cause code:
|
||||
|
||||
```perl
|
||||
elsif (my $cl = $env->{CONTENT_LENGTH}) {
|
||||
my $buf = Plack::TempBuffer->new($cl);
|
||||
while ($cl > 0) {
|
||||
my($chunk, $read) = $get_chunk->();
|
||||
if ( !defined $read || $read == 0 ) {
|
||||
die "Read error: $!\n";
|
||||
}
|
||||
$cl -= $read;
|
||||
$buf->print($chunk);
|
||||
}
|
||||
$env->{'psgi.input'} = $buf->rewind;
|
||||
}
|
||||
```
|
||||
|
||||
### B. Direct Starman accepts invalid `Content-Length` values
|
||||
|
||||
`HTTP::Parser::XS` copies `Content-Length` as a header value and Starman relies on Perl numeric coercion instead of strict decimal validation.
|
||||
|
||||
Parser-level results:
|
||||
|
||||
```text
|
||||
CL=-1 ret=48 CONTENT_LENGTH=-1
|
||||
CL=1x ret=48 CONTENT_LENGTH=1x
|
||||
CL=1e9 ret=49 CONTENT_LENGTH=1e9
|
||||
CL=+1 ret=48 CONTENT_LENGTH=+1
|
||||
```
|
||||
|
||||
Confirmed direct behavior:
|
||||
|
||||
- `Content-Length: -1` to direct `:5000` returns `200 OK` for `/nix-cache-info`.
|
||||
- `Content-Length: 1x` with a one-byte body to direct `:5000` returns `200 OK`.
|
||||
|
||||
Confirmed Caddy behavior:
|
||||
|
||||
- Caddy rejects these invalid content lengths with `400 Bad Request`, so this is currently direct-port-only unless another frontend forwards such requests.
|
||||
|
||||
### C. `%00` in path actively changes the routed endpoint
|
||||
|
||||
As noted below, `%00` truncates `PATH_INFO`. This is not just parser API behavior: both direct Starman and Caddy route `GET /nix-cache-info%00suffix HTTP/1.1` to the `/nix-cache-info` handler and return `200 OK`.
|
||||
|
||||
Current impact is endpoint confusion rather than data exposure because Caddy has no path-level allow/deny rules and the suffix does not select a protected app route. It would become a bypass if path filtering were added at Caddy or middleware while Starman still receives the raw encoded target.
|
||||
|
||||
### D. Missing `/log/...` returns `200 OK` with an empty body
|
||||
|
||||
Requests for a non-existent valid-looking log path return `200 OK` and an empty body through both direct Starman and Caddy:
|
||||
|
||||
```text
|
||||
GET /log/00000000000000000000000000000000-test -> HTTP/1.1 200 OK
|
||||
```
|
||||
|
||||
This is not data exposure, but it is undesired behavior for clients and monitoring because errors from `nix log` are not converted into HTTP errors. The route streams the child stdout without checking exit status.
|
||||
|
||||
## Candidate Entry Points
|
||||
|
||||
### 1. Percent-decoded `PATH_INFO` before routing
|
||||
|
||||
`HTTP::Parser::XS::parse_http_request` stores the original target in `REQUEST_URI`, then percent-decodes the path portion into `PATH_INFO` before `nix-serve.psgi` sees it:
|
||||
|
||||
- `XS.xs:186-201`
|
||||
- `nix-serve.psgi:24`
|
||||
|
||||
This makes encoded delimiters and control bytes relevant to app routing. The app regexes are written as if `$path` is a normal textual URL path, but it is already decoded by the server parser.
|
||||
|
||||
Most interesting subcase: `%00`. In `XS.xs:136-144`, decoded values are stored with `newSVpv(decoded, 0)`. `newSVpv(..., 0)` treats the decoded buffer as a C string, so an embedded NUL produced from `%00` truncates the Perl scalar. A request target such as `/nix-cache-info%00suffix` becomes `PATH_INFO == "/nix-cache-info"` at the PSGI layer.
|
||||
|
||||
Confirmed with the packaged parser:
|
||||
|
||||
```text
|
||||
ret=50
|
||||
PATH_INFO=/nix-cache-info len=15
|
||||
REQUEST_URI=/nix-cache-info%00suffix len=24
|
||||
```
|
||||
|
||||
Confirmed through both local Starman and the Caddy reverse proxy: `GET /nix-cache-info%00suffix HTTP/1.1` returns the `/nix-cache-info` response.
|
||||
|
||||
Impact:
|
||||
|
||||
- Route suffix bypasses if any future route-level filtering is added before Starman decoding is understood.
|
||||
- Caddy/Starman disagreement: Caddy forwards the raw target, while Starman truncates `PATH_INFO` after decoding.
|
||||
- Possible endpoint confusion for `/nar/...` or `/log/...` where suffix data is invisible to the app but present in `REQUEST_URI` and access logs.
|
||||
|
||||
### 2. Out-of-bounds read on malformed percent escapes
|
||||
|
||||
`url_decode` in `XS.xs:97-128` scans for `%`, allocates `len - 1`, then reads `s[i + 1]` and `s[i + 2]` without first checking that two bytes remain:
|
||||
|
||||
```c
|
||||
if ((hi = hex_decode(s[i + 1])) == -1
|
||||
|| (lo = hex_decode(s[i + 2])) == -1) {
|
||||
```
|
||||
|
||||
For a path ending in `%` or `%0`, this reads past the logical end of the Perl string. Perl SV buffers are usually NUL-terminated, so this is likely a small out-of-bounds read rather than an immediate crash, but it is still memory-unsafe C on request-controlled input. Packaged-parser behavior for `/%`, `/%0`, `/%GG`, and `/%0G` is `ret=-1` with no env entries; ASAN/debug-allocator validation is still needed for the actual memory read.
|
||||
|
||||
Impact to investigate:
|
||||
|
||||
- Whether ASAN or a hardened allocator catches reads for trailing `%` / `%0`.
|
||||
- Whether Caddy rejects those targets before forwarding; direct `:5000` remains exposed locally.
|
||||
|
||||
### 3. `/log/...` prefix regex and subprocess spawning
|
||||
|
||||
`nix-serve.psgi:82-86` matches logs with:
|
||||
|
||||
```perl
|
||||
elsif ($path =~ /^\/log\/([0-9a-z]+-[0-9a-zA-Z\+\-\.\_\?\=]+)/) {
|
||||
```
|
||||
|
||||
The regex is not anchored at the end. Any path beginning with a valid-looking store basename is accepted, and the suffix is ignored. The route then runs `nix log $storePath` for the captured value without checking it with `queryPathFromHashPart` or `queryPathInfo` first.
|
||||
|
||||
There is no shell injection because `open` is called with an argument list, not a shell string. The interesting angle is resource use and Nix behavior on attacker-chosen valid-looking store paths.
|
||||
|
||||
Local command timing for a missing valid-looking path:
|
||||
|
||||
```text
|
||||
$ time nix --extra-experimental-features nix-command log /nix/store/00000000000000000000000000000000-test
|
||||
error: build log of '/nix/store/00000000000000000000000000000000-test' is not available
|
||||
real 0m1.892s
|
||||
```
|
||||
|
||||
That is enough per request to make `/log/...` a plausible low-rate process/CPU DoS surface, especially because the app does not validate the path against the store before spawning `nix log`.
|
||||
|
||||
Impact:
|
||||
|
||||
- CPU/process exhaustion from repeated `nix log` subprocesses.
|
||||
- Missing paths still cost roughly 1.9s locally in a direct command test.
|
||||
- Whether ignored suffixes create Caddy/Starman/app log ambiguity.
|
||||
|
||||
### 4. `/nar/...` subprocess fan-out
|
||||
|
||||
Both NAR routes spawn a `nix store dump-path` process per request:
|
||||
|
||||
- Checked route: `nix-serve.psgi:58-68`
|
||||
- Legacy unchecked-hash route: `nix-serve.psgi:72-79`
|
||||
|
||||
The checked route validates that the requested NAR hash matches current path info. The legacy route only checks the hash prefix maps to a path and then dumps it.
|
||||
|
||||
Impact to investigate:
|
||||
|
||||
- Bandwidth/process DoS by repeatedly requesting large store paths.
|
||||
- Whether the legacy route should be disabled in this deployment.
|
||||
- Whether Caddy should apply rate limits or buffering constraints.
|
||||
|
||||
### 5. Request body parser inconsistencies in `HTTP::Entity::Parser`
|
||||
|
||||
`nix-serve.psgi` does not call `HTTP::Entity::Parser`, so this is probably not reachable through the current app unless Starman/Plack middleware invokes it. It is still in the service closure and should be treated as a package-level finding.
|
||||
|
||||
Potential issues:
|
||||
|
||||
- `HTTP::Entity::Parser.pm:76-90`: if both `Content-Length` and `Transfer-Encoding: chunked` exist, `Content-Length` wins. RFC 7230 says transfer coding overrides content length; this can create request-smuggling-style disagreement when another component follows the standard.
|
||||
- `HTTP::Entity::Parser.pm:77-88`: `CONTENT_LENGTH` is not strictly parsed as decimal digits. Perl numeric coercion can accept weird values like `10foo`, scientific notation, or negative values differently from other components.
|
||||
- `HTTP::Entity::Parser.pm:102-115`: chunk header parsing accepts `^(([0-9a-fA-F]+).*\r\n)` and does not require the CRLF after chunk data; it merely tries to remove it with `s/^\r\n//`. Malformed chunk bodies can be accepted with parser state disagreement.
|
||||
- `UrlEncoded.pm` and `JSON.pm` accumulate the full body in memory before final parsing. Large bodies are memory DoS if an app registers these parsers without external limits.
|
||||
- `MultiPart.pm` writes uploaded file parts to temp files and accumulates non-file fields in memory. There are no per-field, per-file, part-count, or aggregate limits here.
|
||||
|
||||
## Initial Priority
|
||||
|
||||
1. Decide whether to block `%00` at Caddy or patch/replace `HTTP::Parser::XS` path decoding.
|
||||
2. Validate malformed percent escape behavior under ASAN or a debug allocator if practical.
|
||||
3. Inspect `nix log` behavior for missing/attacker-chosen valid-looking store paths and decide whether `/log` needs validation/rate-limiting.
|
||||
4. Decide whether the legacy `/nar/<hash>.nar` route is still needed.
|
||||
5. Treat `HTTP::Entity::Parser` as lower priority for this app unless a middleware path is found that parses request bodies.
|
||||
|
|
@ -107,6 +107,7 @@
|
|||
home-manager.useGlobalPkgs = true;
|
||||
home-manager.useUserPackages = true;
|
||||
home-manager.users.dan = import ./servers/ui-mode/home.nix;
|
||||
home-manager.backupFileExtension = "backup";
|
||||
networking.hostName = "fern";
|
||||
imports = [ ./servers/fern/hardware-configuration.nix ];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,10 @@ in
|
|||
capSysAdmin = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
programs.steam.extraPackages = [ pkgs.hidapi ];
|
||||
programs.steam.extraPackages = with pkgs; [
|
||||
hidapi
|
||||
pulseaudio
|
||||
];
|
||||
|
||||
nixpkgs.config.permittedInsecurePackages = [
|
||||
"olm-3.2.16"
|
||||
|
|
@ -94,6 +97,11 @@ in
|
|||
|
||||
time.timeZone = lib.mkForce "Europe/Prague";
|
||||
i18n.defaultLocale = "en_US.UTF-8";
|
||||
i18n.supportedLocales = [
|
||||
"cs_CZ.UTF-8/UTF-8"
|
||||
"en_GB.UTF-8/UTF-8"
|
||||
"en_US.UTF-8/UTF-8"
|
||||
];
|
||||
i18n.extraLocaleSettings = {
|
||||
LC_ADDRESS = "cs_CZ.UTF-8";
|
||||
LC_IDENTIFICATION = "cs_CZ.UTF-8";
|
||||
|
|
@ -187,6 +195,7 @@ in
|
|||
|
||||
# Other defaults are set in home.nix
|
||||
# environment.sessionVariables.DEFAULT_BROWSER = "firefox";
|
||||
environment.sessionVariables.NIXOS_OZONE_WL = "1";
|
||||
|
||||
# programs.firefox.enable = true;
|
||||
nix.settings = {
|
||||
|
|
@ -238,15 +247,33 @@ in
|
|||
"hyprland"
|
||||
"gtk"
|
||||
];
|
||||
common."org.freedesktop.impl.portal.ScreenCast" = [ "hyprland" ];
|
||||
common."org.freedesktop.impl.portal.RemoteDesktop" = [ "hypr-kdeconnect" ];
|
||||
hyprland = {
|
||||
default = [
|
||||
"hyprland"
|
||||
"gtk"
|
||||
];
|
||||
"org.freedesktop.impl.portal.FileChooser" = [ "gtk" ];
|
||||
"org.freedesktop.impl.portal.ScreenCast" = [ "hyprland" ];
|
||||
"org.freedesktop.impl.portal.RemoteDesktop" = [ "hypr-kdeconnect" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
systemd.user.services.plasma-xdg-desktop-portal-kde = {
|
||||
unitConfig = {
|
||||
Description = "Xdg Desktop Portal For KDE";
|
||||
PartOf = "graphical-session.target";
|
||||
After = "plasma-core.target";
|
||||
ConditionEnvironment = "XDG_CURRENT_DESKTOP=KDE";
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.kdePackages.xdg-desktop-portal-kde}/libexec/xdg-desktop-portal-kde";
|
||||
BusName = "org.freedesktop.impl.portal.desktop.kde";
|
||||
Slice = "session.slice";
|
||||
Restart = "no";
|
||||
};
|
||||
};
|
||||
programs.dank-material-shell.greeter = {
|
||||
enable = true;
|
||||
compositor.name = "hyprland"; # "niri" or "hyprland" or "sway"
|
||||
|
|
|
|||
|
|
@ -52,6 +52,15 @@ let
|
|||
${codexbar.packages.${pkgs.system}.default}/bin/codexbar "$@"
|
||||
'';
|
||||
};
|
||||
vesktopWrapped = pkgs.symlinkJoin {
|
||||
name = "vesktop-pipewire";
|
||||
paths = [ pkgs.vesktop ];
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
postBuild = ''
|
||||
wrapProgram $out/bin/vesktop \
|
||||
--add-flags "--enable-features=WebRTCPipeWireCapturer,WaylandWindowDecorations"
|
||||
'';
|
||||
};
|
||||
# system = stdenv.hostPlatform.system;
|
||||
in
|
||||
{
|
||||
|
|
@ -67,6 +76,9 @@ in
|
|||
stateVersion = "25.11";
|
||||
|
||||
packages = with pkgs; [
|
||||
firefox
|
||||
unrar
|
||||
wine
|
||||
codexbarWrapped
|
||||
codex
|
||||
jellyfin-desktop
|
||||
|
|
@ -205,7 +217,7 @@ in
|
|||
#rofi-wayland
|
||||
rofi
|
||||
discord
|
||||
vesktop
|
||||
vesktopWrapped
|
||||
spotify
|
||||
spicetify-cli
|
||||
meslo-lgs-nf
|
||||
|
|
@ -260,6 +272,7 @@ in
|
|||
mpv
|
||||
|
||||
heroic
|
||||
gamescope
|
||||
heaptrack
|
||||
#cinny-desktop
|
||||
gping
|
||||
|
|
|
|||
96
steam-heroic-shortcuts.md
Normal file
96
steam-heroic-shortcuts.md
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
# Steam Heroic Direct Shortcuts
|
||||
|
||||
Reference for the non-declarative Steam shortcuts added for Heroic-managed games.
|
||||
These live in Steam config files, not Nix, so this file records the important state for future recovery.
|
||||
|
||||
## Files
|
||||
|
||||
- Steam shortcuts: `/home/dan/.local/share/Steam/userdata/238310127/config/shortcuts.vdf`
|
||||
- Steam compatibility mapping: `/home/dan/.local/share/Steam/config/config.vdf`
|
||||
- Steam compatdata: `/home/dan/.local/share/Steam/steamapps/compatdata/`
|
||||
- Heroic game config: `/home/dan/.config/heroic/GamesConfig/`
|
||||
|
||||
## Steam Shortcuts
|
||||
|
||||
All direct shortcuts are configured to use `DW-Proton Latest`.
|
||||
|
||||
| Name | App ID | Exe | StartDir | Launch options |
|
||||
| --- | ---: | --- | --- | --- |
|
||||
| `Arknights: Endfield (Direct Launcher)` | `3532200938` | `/home/dan/Games/Heroic/ArknightsEndfieldgowoU/Launcher.exe` | `/home/dan/Games/Heroic/ArknightsEndfieldgowoU` | empty |
|
||||
| `Arknights: Endfield (Direct Game)` | `2506976826` | `/home/dan/Games/Heroic/ArknightsEndfieldgowoU/games/EndField Game/Endfield.exe` | `/home/dan/Games/Heroic/ArknightsEndfieldgowoU/games/EndField Game` | empty |
|
||||
| `Zenless Zone Zero (Direct Launcher)` | `2568476331` | `/home/dan/Games/Heroic/ZenlessZoneZero/launcher_epic.exe` | `/home/dan/Games/Heroic/ZenlessZoneZero` | `UMU_ID=umu-zenlesszonezero UMU_USE_STEAM=1 WINE_DISABLE_VULKAN_OPWR=1 %command% {enable_pay:true}` |
|
||||
| `Zenless Zone Zero (Direct Game)` | `4264951319` | `/home/dan/Games/Heroic/ZenlessZoneZero/games/ZenlessZoneZero Game/ZenlessZoneZero.exe` | `/home/dan/Games/Heroic/ZenlessZoneZero/games/ZenlessZoneZero Game` | `UMU_ID=umu-zenlesszonezero UMU_USE_STEAM=1 WINE_DISABLE_VULKAN_OPWR=1 %command%` |
|
||||
|
||||
The original Heroic-generated shortcuts were left in place:
|
||||
|
||||
| Name | App ID | Exe | Launch options |
|
||||
| --- | ---: | --- | --- |
|
||||
| `Zenless Zone Zero` | `2928100415` | `heroic` | `--no-gui --no-sandbox "heroic://launch?appName=525aa0efd70f4399b9f64bcd2a5b38c7&runner=legendary"` |
|
||||
| `Arknights: Endfield` | `2465091319` | `heroic` | `--no-gui --no-sandbox "heroic://launch?appName=bcd55b0d87c245dd867f5b1bd496f1df&runner=legendary"` |
|
||||
|
||||
## Compatibility Mapping
|
||||
|
||||
The app IDs above were added under `InstallConfigStore.Software.Valve.Steam.CompatToolMapping` in Steam's `config.vdf`:
|
||||
|
||||
```vdf
|
||||
"CompatToolMapping"
|
||||
{
|
||||
"3532200938"
|
||||
{
|
||||
"name" "DW-Proton Latest"
|
||||
"config" ""
|
||||
"priority" "250"
|
||||
}
|
||||
"2506976826"
|
||||
{
|
||||
"name" "DW-Proton Latest"
|
||||
"config" ""
|
||||
"priority" "250"
|
||||
}
|
||||
"2568476331"
|
||||
{
|
||||
"name" "DW-Proton Latest"
|
||||
"config" ""
|
||||
"priority" "250"
|
||||
}
|
||||
"4264951319"
|
||||
{
|
||||
"name" "DW-Proton Latest"
|
||||
"config" ""
|
||||
"priority" "250"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Prefix Links
|
||||
|
||||
The direct Steam shortcuts use the existing Heroic prefixes by symlinking each shortcut's `pfx` directory:
|
||||
|
||||
```sh
|
||||
ln -s "/home/dan/Games/Heroic/Prefixes/default/Zenless Zone Zero" \
|
||||
"/home/dan/.local/share/Steam/steamapps/compatdata/2568476331/pfx"
|
||||
|
||||
ln -s "/home/dan/Games/Heroic/Prefixes/default/Zenless Zone Zero" \
|
||||
"/home/dan/.local/share/Steam/steamapps/compatdata/4264951319/pfx"
|
||||
|
||||
ln -s "/home/dan/Games/Heroic/Prefixes/default/Arknights Endfield" \
|
||||
"/home/dan/.local/share/Steam/steamapps/compatdata/3532200938/pfx"
|
||||
|
||||
ln -s "/home/dan/Games/Heroic/Prefixes/default/Arknights Endfield" \
|
||||
"/home/dan/.local/share/Steam/steamapps/compatdata/2506976826/pfx"
|
||||
```
|
||||
|
||||
If Steam already created a fresh prefix, move it aside before creating the symlink:
|
||||
|
||||
```sh
|
||||
mv "/home/dan/.local/share/Steam/steamapps/compatdata/4264951319/pfx" \
|
||||
"/home/dan/.local/share/Steam/steamapps/compatdata/4264951319/pfx.steam-empty-bak"
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- `gamescope` is installed declaratively in `servers/ui-mode/home.nix` for optional testing.
|
||||
- `Zenless Zone Zero (Direct Game)` is the most promising shortcut: the game process starts and Steam starts `gameoverlayui` for it.
|
||||
- The Heroic-generated `heroic://launch` shortcuts can start the games, but Steam Overlay/Input may not attach because Steam tracks Heroic/Electron rather than the final game process.
|
||||
- Zenless launcher mode needs fresh Epic exchange-code arguments from Heroic/Legendary, so the direct launcher shortcut may not be reliable.
|
||||
- A `wine64-preloader`/`rpcss.exe` `SIGSYS` coredump was seen during startup, but the game continued and overlay was started; treat it as non-fatal unless the game crashes.
|
||||
Loading…
Reference in a new issue