dan/dotfiles
1
0
Fork 0
mirror of https://github.com/danbulant/dotfiles synced 2026-09-17 13:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update llms

Browse source
This commit is contained in:
Daniel Bulant 2026-08-16 12:47:57 +02:00
parent 7c2675f0e9
commit 4613a5a905
No known key found for this signature in database
1 changed files with 23 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
servers/fern/configuration.nix
View file

@ -44,25 +44,23 @@ let
metalSupport = false; metalSupport = false;
blasSupport = true; blasSupport = true;
}).overrideAttrs }).overrideAttrs
(prevAttrs: rec { (_: rec {
preConfigure = '' version = "10434";
export NIX_ENFORCE_NO_NATIVE=0
${prevAttrs.preConfigure or ""}
'';
version = "8999";
src = pkgs.fetchFromGitHub { src = pkgs.fetchFromGitHub {
owner = "ggml-org"; owner = "ggml-org";
repo = "llama.cpp"; repo = "llama.cpp";
tag = "b${version}"; tag = "b${version}";
hash = "sha256-EgJ3Die/WpVm9dtQ2kwXoV4RAWNY9x7lT4wun79qqCI="; hash = "sha256-Sz0kW1q91YzdrKbZUqMbFJ0DLZrzARSGheUrtCKcoQo=";
leaveDotGit = true;
postFetch = ''
git -C "$out" rev-parse --short HEAD > $out/COMMIT
find "$out" -name .git -print0 | xargs -0 rm -rf
'';
}; };
npmRoot = "tools/server/webui"; npmRoot = "tools/ui";
npmDepsHash = "sha256-k62LIbyY2DXvs7XXbX0lNPiYxuYzeJUyQtS4eA+68f8="; npmDepsHash = "sha256-2Q7XhaLAArmviOLdQsNbYTfdyDE5pW9lR26cRHEVl9k=";
preConfigure = ''
export NIX_ENFORCE_NO_NATIVE=0
prependToVar cmakeFlags "-DLLAMA_BUILD_COMMIT:STRING=7e4c0a96"
pushd ${npmRoot}
LLAMA_BUILD_NUMBER=${version} npm run build
popd
'';
cmakeFlags = with pkgs.lib; [ cmakeFlags = with pkgs.lib; [
# -march=native is non-deterministic; override with platform-specific flags if needed # -march=native is non-deterministic; override with platform-specific flags if needed
(cmakeBool "GGML_NATIVE" true) (cmakeBool "GGML_NATIVE" true)
@ -73,7 +71,6 @@ let
(cmakeBool "BUILD_SHARED_LIBS" true) (cmakeBool "BUILD_SHARED_LIBS" true)
# (cmakeBool "GGML_BLAS" false) # (cmakeBool "GGML_BLAS" false)
(cmakeBool "GGML_LTO" true) (cmakeBool "GGML_LTO" true)
(cmakeBool "GGML_CLBLAST" true)
(cmakeBool "GGML_CUDA" true) (cmakeBool "GGML_CUDA" true)
(cmakeBool "GGML_CUDA_GRAPHS" true) (cmakeBool "GGML_CUDA_GRAPHS" true)
(cmakeBool "GGML_CUDA_F16" true) (cmakeBool "GGML_CUDA_F16" true)
@ -218,15 +215,16 @@ in
--alias "unsloth/qwen" \ --alias "unsloth/qwen" \
--no-webui \ --no-webui \
--ctx-size 131072 \ --ctx-size 131072 \
--gpu-layers auto \
--fit on --fit-ctx 131072 --fit-target 256 \ --fit on --fit-ctx 131072 --fit-target 256 \
--temp 1.0 --top-p 0.95 --top-k 64 \ --temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0 \
--repeat-penalty 1.0 \ --repeat-penalty 1.0 \
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--flash-attn on \ --flash-attn on \
--batch-size 1024 --ubatch-size 512 \ --batch-size 1024 --ubatch-size 512 \
--threads 12 --threads-batch 12 \ --threads 12 --threads-batch 12 \
--no-mmap --mlock \ --load-mode mlock \
--parallel 1 --prio 2 --no-warmup --jinja --parallel 1 --no-warmup --jinja
''; '';
models_dir = "\${env.HOME}/models"; models_dir = "\${env.HOME}/models";
}; };
@ -238,6 +236,12 @@ in
"qwen3.6-35B-A3B" = { "qwen3.6-35B-A3B" = {
cmd = "\${llama} -m /home/dan/.lmstudio/models/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"; cmd = "\${llama} -m /home/dan/.lmstudio/models/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
}; };
"qwen3.8-27B-Q2" = {
cmd = "\${llama} -m /home/dan/models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q2_K_XL.gguf --ctx-size 16384 --fit-ctx 16384";
};
"qwen3.8-27B-Q4" = {
cmd = "\${llama} -m /home/dan/models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_0.gguf --ctx-size 100000 --fit-ctx 100000 -ctk q4_0 -ctv q4_0 --ubatch-size 128 --spec-type draft-mtp --spec-draft-n-max 3";
};
"gemma-4-26B-A4B" = { "gemma-4-26B-A4B" = {
cmd = "\${llama} -m /home/dan/.lmstudio/models/lmstudio-community/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf"; cmd = "\${llama} -m /home/dan/.lmstudio/models/lmstudio-community/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf";
}; };
@ -259,6 +263,7 @@ in
DynamicUser = pkgs.lib.mkForce false; DynamicUser = pkgs.lib.mkForce false;
User = pkgs.lib.mkForce "dan"; User = pkgs.lib.mkForce "dan";
Group = pkgs.lib.mkForce "users"; # or dan's primary group Group = pkgs.lib.mkForce "users"; # or dan's primary group
LimitMEMLOCK = "infinity";
ExecStart = lib.mkForce '' ExecStart = lib.mkForce ''
${lib.getExe pkgs.llama-swap} --listen 0.0.0.0:${toString config.services.llama-swap.port} --config ${ ${lib.getExe pkgs.llama-swap} --listen 0.0.0.0:${toString config.services.llama-swap.port} --config ${
(pkgs.formats.yaml { }).generate "config.yaml" config.services.llama-swap.settings (pkgs.formats.yaml { }).generate "config.yaml" config.services.llama-swap.settings