fixup local-llama service
This commit is contained in:
@@ -3,7 +3,7 @@ MODEL_FILE=Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
|
||||
N_GPU_LAYERS="99"
|
||||
|
||||
# Доп. ключи docker-compose (опционально; дефолты заданы в compose)
|
||||
CTX_SIZE=175000
|
||||
CTX_SIZE=170000
|
||||
THREADS=12
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=q8_0
|
||||
|
||||
+5
-5
@@ -4,11 +4,11 @@ After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=grayhook
|
||||
Group=grayhook
|
||||
WorkingDirectory=/home/grayhook/Documents/llama-space
|
||||
EnvironmentFile=/home/grayhook/Documents/llama-space/.env
|
||||
ExecStart=/home/grayhook/Documents/llama-space/local-llama.sh
|
||||
User=bazzite
|
||||
Group=bazzite
|
||||
WorkingDirectory=/home/bazzite/llama-space
|
||||
EnvironmentFile=/home/bazzite/llama-space/.env
|
||||
ExecStart=/home/bazzite/llama-space/local-llama.sh
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
|
||||
+4
-1
@@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
export PATH=/usr/local/cuda-13.3/bin${PATH:+:${PATH}}
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-13.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
/home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
|
||||
@@ -8,7 +11,7 @@ set -euo pipefail
|
||||
--port 8080 \
|
||||
-ngl "${N_GPU_LAYERS}" \
|
||||
--n-cpu-moe "${N_CPU_MOE}" \
|
||||
--spec-type draft-mtp --spec-draft-n-max 3 \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
-c "${CTX_SIZE}" \
|
||||
-np 1 \
|
||||
-fa on \
|
||||
|
||||
Reference in New Issue
Block a user