fixup local-llama service
This commit is contained in:
@@ -3,7 +3,7 @@ MODEL_FILE=Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
|
|||||||
N_GPU_LAYERS="99"
|
N_GPU_LAYERS="99"
|
||||||
|
|
||||||
# Доп. ключи docker-compose (опционально; дефолты заданы в compose)
|
# Доп. ключи docker-compose (опционально; дефолты заданы в compose)
|
||||||
CTX_SIZE=175000
|
CTX_SIZE=170000
|
||||||
THREADS=12
|
THREADS=12
|
||||||
CACHE_TYPE_K=q8_0
|
CACHE_TYPE_K=q8_0
|
||||||
CACHE_TYPE_V=q8_0
|
CACHE_TYPE_V=q8_0
|
||||||
|
|||||||
+5
-5
@@ -4,11 +4,11 @@ After=network.target
|
|||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
User=grayhook
|
User=bazzite
|
||||||
Group=grayhook
|
Group=bazzite
|
||||||
WorkingDirectory=/home/grayhook/Documents/llama-space
|
WorkingDirectory=/home/bazzite/llama-space
|
||||||
EnvironmentFile=/home/grayhook/Documents/llama-space/.env
|
EnvironmentFile=/home/bazzite/llama-space/.env
|
||||||
ExecStart=/home/grayhook/Documents/llama-space/local-llama.sh
|
ExecStart=/home/bazzite/llama-space/local-llama.sh
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
|
|
||||||
|
|||||||
+4
-1
@@ -1,5 +1,8 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
export PATH=/usr/local/cuda-13.3/bin${PATH:+:${PATH}}
|
||||||
|
export LD_LIBRARY_PATH=/usr/local/cuda-13.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
/home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
|
/home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
|
||||||
@@ -8,7 +11,7 @@ set -euo pipefail
|
|||||||
--port 8080 \
|
--port 8080 \
|
||||||
-ngl "${N_GPU_LAYERS}" \
|
-ngl "${N_GPU_LAYERS}" \
|
||||||
--n-cpu-moe "${N_CPU_MOE}" \
|
--n-cpu-moe "${N_CPU_MOE}" \
|
||||||
--spec-type draft-mtp --spec-draft-n-max 3 \
|
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||||
-c "${CTX_SIZE}" \
|
-c "${CTX_SIZE}" \
|
||||||
-np 1 \
|
-np 1 \
|
||||||
-fa on \
|
-fa on \
|
||||||
|
|||||||
Reference in New Issue
Block a user