fixup local-llama service
This commit is contained in:
+4
-1
@@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
export PATH=/usr/local/cuda-13.3/bin${PATH:+:${PATH}}
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-13.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
/home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
|
||||
@@ -8,7 +11,7 @@ set -euo pipefail
|
||||
--port 8080 \
|
||||
-ngl "${N_GPU_LAYERS}" \
|
||||
--n-cpu-moe "${N_CPU_MOE}" \
|
||||
--spec-type draft-mtp --spec-draft-n-max 3 \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
-c "${CTX_SIZE}" \
|
||||
-np 1 \
|
||||
-fa on \
|
||||
|
||||
Reference in New Issue
Block a user