From 4613a5a90542dd2611af750e7a1fa5c990c83bb8 Mon Sep 17 00:00:00 2001
From: Daniel Bulant <danbulant@gmail.com>
Date: Sun, 16 Aug 2026 12:47:57 +0200
Subject: [PATCH] update llms

---
 servers/fern/configuration.nix | 41 +++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/servers/fern/configuration.nix b/servers/fern/configuration.nix
index bd46b42..747659c 100644
--- a/servers/fern/configuration.nix
+++ b/servers/fern/configuration.nix
@@ -44,25 +44,23 @@ let
       metalSupport = false;
       blasSupport = true;
     }).overrideAttrs
-      (prevAttrs: rec {
-        preConfigure = ''
-          export NIX_ENFORCE_NO_NATIVE=0
-          ${prevAttrs.preConfigure or ""}
-        '';
-        version = "8999";
+      (_: rec {
+        version = "10434";
         src = pkgs.fetchFromGitHub {
           owner = "ggml-org";
           repo = "llama.cpp";
           tag = "b${version}";
-          hash = "sha256-EgJ3Die/WpVm9dtQ2kwXoV4RAWNY9x7lT4wun79qqCI=";
-          leaveDotGit = true;
-          postFetch = ''
-            git -C "$out" rev-parse --short HEAD > $out/COMMIT
-            find "$out" -name .git -print0 | xargs -0 rm -rf
-          '';
+          hash = "sha256-Sz0kW1q91YzdrKbZUqMbFJ0DLZrzARSGheUrtCKcoQo=";
         };
-        npmRoot = "tools/server/webui";
-        npmDepsHash = "sha256-k62LIbyY2DXvs7XXbX0lNPiYxuYzeJUyQtS4eA+68f8=";
+        npmRoot = "tools/ui";
+        npmDepsHash = "sha256-2Q7XhaLAArmviOLdQsNbYTfdyDE5pW9lR26cRHEVl9k=";
+        preConfigure = ''
+          export NIX_ENFORCE_NO_NATIVE=0
+          prependToVar cmakeFlags "-DLLAMA_BUILD_COMMIT:STRING=7e4c0a96"
+          pushd ${npmRoot}
+          LLAMA_BUILD_NUMBER=${version} npm run build
+          popd
+        '';
         cmakeFlags = with pkgs.lib; [
           # -march=native is non-deterministic; override with platform-specific flags if needed
           (cmakeBool "GGML_NATIVE" true)
@@ -73,7 +71,6 @@ let
           (cmakeBool "BUILD_SHARED_LIBS" true)
           # (cmakeBool "GGML_BLAS" false)
           (cmakeBool "GGML_LTO" true)
-          (cmakeBool "GGML_CLBLAST" true)
           (cmakeBool "GGML_CUDA" true)
           (cmakeBool "GGML_CUDA_GRAPHS" true)
           (cmakeBool "GGML_CUDA_F16" true)
@@ -218,15 +215,16 @@ in
           --alias "unsloth/qwen" \
           --no-webui \
           --ctx-size 131072 \
+          --gpu-layers auto \
           --fit on --fit-ctx 131072 --fit-target 256 \
-          --temp 1.0 --top-p 0.95 --top-k 64 \
+          --temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0 \
           --repeat-penalty 1.0 \
           -ctk q8_0 -ctv q8_0 \
           --flash-attn on \
           --batch-size 1024 --ubatch-size 512 \
           --threads 12 --threads-batch 12 \
-          --no-mmap --mlock \
-          --parallel 1 --prio 2 --no-warmup --jinja
+          --load-mode mlock \
+          --parallel 1 --no-warmup --jinja
         '';
         models_dir = "\${env.HOME}/models";
       };
@@ -238,6 +236,12 @@ in
         "qwen3.6-35B-A3B" = {
           cmd = "\${llama} -m /home/dan/.lmstudio/models/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
         };
+        "qwen3.8-27B-Q2" = {
+          cmd = "\${llama} -m /home/dan/models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q2_K_XL.gguf --ctx-size 16384 --fit-ctx 16384";
+        };
+        "qwen3.8-27B-Q4" = {
+          cmd = "\${llama} -m /home/dan/models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_0.gguf --ctx-size 100000 --fit-ctx 100000 -ctk q4_0 -ctv q4_0 --ubatch-size 128 --spec-type draft-mtp --spec-draft-n-max 3";
+        };
         "gemma-4-26B-A4B" = {
           cmd = "\${llama} -m /home/dan/.lmstudio/models/lmstudio-community/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf";
         };
@@ -259,6 +263,7 @@ in
       DynamicUser = pkgs.lib.mkForce false;
       User = pkgs.lib.mkForce "dan";
       Group = pkgs.lib.mkForce "users"; # or dan's primary group
+      LimitMEMLOCK = "infinity";
       ExecStart = lib.mkForce ''
         ${lib.getExe pkgs.llama-swap} --listen 0.0.0.0:${toString config.services.llama-swap.port} --config ${
           (pkgs.formats.yaml { }).generate "config.yaml" config.services.llama-swap.settings