tune and run local with turboquant

2026-06-22 10:37:43 +07:00
parent a0713b260b
commit 2b5977b605
3 changed files with 39 additions and 6 deletions
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -a            # Automatically export all variables defined next
+source .env       # Load the file
+set +a            # Disable automatic export
+
+sudo ulimit -l unlimited
+
+taskset -c 0-11 /home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
+  -m "models/${MODEL_FILE}" \
+  --host 0.0.0.0 \
+  --port 8080 \
+  -ngl "${N_GPU_LAYERS}" \
+  --n-cpu-moe "${N_CPU_MOE}" \
+  --spec-type draft-mtp --spec-draft-n-max 3 \
+  -c "${CTX_SIZE}" \
+  -np 1 \
+  -fa on \
+  --cache-type-k "turbo4" \
+  --cache-type-v "turbo3" \
+  --no-mmap \
+  --mlock \
+  --ctx-checkpoints 1 \
+  --cache-ram -1 \
+  --jinja \
+  --reasoning on \
+  --reasoning-budget -1 \
+  -b 2048 \
+  -ub 2048 \
+  --threads "${THREADS}"