From 2b5977b6055baaf7bdd59f2c9fe5dca699d314da Mon Sep 17 00:00:00 2001
From: grayhook <s@marinkevich.ru>
Date: Mon, 22 Jun 2026 10:37:43 +0700
Subject: [PATCH] tune and run local with turboquant

---
 .env            |  8 ++++----
 local-llama.sh  | 30 ++++++++++++++++++++++++++++++
 podman-llama.sh |  7 +++++--
 3 files changed, 39 insertions(+), 6 deletions(-)
 create mode 100755 local-llama.sh

diff --git a/.env b/.env
index 16bcfac..e51fde6 100644
--- a/.env
+++ b/.env
@@ -1,10 +1,10 @@
-MODEL_FILE=Qwen3.6-35B-A3B-MXFP4_MOE.gguf
+MODEL_FILE=Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
 # Для -1 в .env лучше кавычки — иначе часть парсеров .env ломается на ведущем минусе.
-N_GPU_LAYERS="-1"
+N_GPU_LAYERS="99"
 
 # Доп. ключи docker-compose (опционально; дефолты заданы в compose)
-CTX_SIZE=65536
-THREADS=8
+CTX_SIZE=175000
+THREADS=12
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=q8_0
 
diff --git a/local-llama.sh b/local-llama.sh
new file mode 100755
index 0000000..16f5757
--- /dev/null
+++ b/local-llama.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -a            # Automatically export all variables defined next
+source .env       # Load the file
+set +a            # Disable automatic export
+
+sudo ulimit -l unlimited
+
+taskset -c 0-11 /home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
+  -m "models/${MODEL_FILE}" \
+  --host 0.0.0.0 \
+  --port 8080 \
+  -ngl "${N_GPU_LAYERS}" \
+  --n-cpu-moe "${N_CPU_MOE}" \
+  --spec-type draft-mtp --spec-draft-n-max 3 \
+  -c "${CTX_SIZE}" \
+  -np 1 \
+  -fa on \
+  --cache-type-k "turbo4" \
+  --cache-type-v "turbo3" \
+  --no-mmap \
+  --mlock \
+  --ctx-checkpoints 1 \
+  --cache-ram -1 \
+  --jinja \
+  --reasoning on \
+  --reasoning-budget -1 \
+  -b 2048 \
+  -ub 2048 \
+  --threads "${THREADS}"
diff --git a/podman-llama.sh b/podman-llama.sh
index 50da5ea..e34f4f7 100755
--- a/podman-llama.sh
+++ b/podman-llama.sh
@@ -30,7 +30,7 @@ fi
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=turbo2}"
 : "${THREADS:=8}"
-: "${IMAGE:=ghcr.io/ggml-org/llama.cpp:server-cuda}"
+: "${IMAGE:=ghcr.io/ggml-org/llama.cpp:server-cuda13}"
 : "${BIND_HOST:=0.0.0.0}"
 
 # Разбить по пробелам для podman (пусто = только CDI-устройство).
@@ -42,6 +42,7 @@ podman run -d \
   --replace \
   --restart no \
   --shm-size 1g \
+  --cpuset-cpus="0-11" \
   --ulimit memlock=-1:-1 \
   -p "${BIND_HOST}:${PORT}:8080" \
   -v "${ROOT}/models:/models:ro" \
@@ -52,7 +53,9 @@ podman run -d \
   -m "/models/${MODEL_FILE}" \
   --host 0.0.0.0 \
   --port 8080 \
+  -ngl "${N_GPU_LAYERS}" \
   --n-cpu-moe "${N_CPU_MOE}" \
+  --spec-type draft-mtp --spec-draft-n-max 3 \
   -c "${CTX_SIZE}" \
   -np 1 \
   -fa on \
@@ -61,7 +64,7 @@ podman run -d \
   --no-mmap \
   --mlock \
   --ctx-checkpoints 1 \
-  --cache-ram 0 \
+  --cache-ram -1 \
   --jinja \
   --reasoning on \
   --reasoning-budget -1 \