migrate to router mode: use models.ini for multi-model support

.env

@@ -1,14 +1,14 @@
-MODEL_FILE=Qwen3.5-9B-MTP-UD-Q4_K_XL.gguf
+MODEL_FILE=Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
 # Для -1 в .env лучше кавычки — иначе часть парсеров .env ломается на ведущем минусе.
-N_GPU_LAYERS="-1"
+N_GPU_LAYERS="99"
 
 # Доп. ключи docker-compose (опционально; дефолты заданы в compose)
-CTX_SIZE=65536
+CTX_SIZE=170000
-THREADS=1
+THREADS=12
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=q8_0
 
 # MoE (Qwen3.5-35B-A3B-Q4_K_M / MXFP4_MOE и т.д.): при OOM на GPU — CPU_MOE=1 или частично N_CPU_MOE=8
 # Для dense (Qwopus 27B, Qwen 9B) держите оба 0.
 CPU_MOE=0
-N_CPU_MOE=0
+N_CPU_MOE=29

docker-compose.yml

@@ -1,21 +1,13 @@
-# llama-cpp-turboquant + CUDA в Docker (Qwen3.x / крупные GGUF на 12 ГБ VRAM: гибрид GPU+CPU).
+# llama.cpp + CUDA в Docker (Qwen3.x / крупные GGUF на 12 ГБ VRAM: гибрид GPU+CPU).
-# Образы собираются локально из клона TheTom/llama-cpp-turboquant (см. docker/).
+# Образ по умолчанию: server-cuda (CUDA 12 в контейнере) — совместим с большинством GPU и драйверов.
-#
+# Альтернатива: server-cuda13 — только если GPU/драйвер тянут CUDA 13; иначе ggml_cuda_init:
-#   CUDA 12 (дефолт) — RTX 30xx / драйвер CUDA 12.x
+#   «forward compatibility was attempted on non supported HW» → оставьте server-cuda.
-#   CUDA 13 — Blackwell (RTX 50xx, sm_120), драйвер ≥ CUDA 13.1:
-#     LLAMA_IMAGE_TAG=server-cuda13 LLAMA_DOCKERFILE=../llama-space/docker/server-cuda13.Dockerfile \
-#       docker-compose build
-#
-# Источник: LLAMA_CPP_SRC (дефолт ../llama-cpp-turboquant).
-# dockerfile — путь относительно context (так требует docker-compose 1.x).
-#
-# Сборка: docker-compose build
-# Запуск: docker-compose up -d
-# Podman: ./podman-llama.sh
-# Проверка: curl http://localhost:8080/health
-#
 # Нужны: драйвер NVIDIA, NVIDIA Container Toolkit, runtime nvidia в Docker.
+#
 # Модель: ./models/ + MODEL_FILE в .env.
+# Запуск: docker compose up -d или docker-compose up -d
+# Podman (одна команда, те же флаги): ./podman-llama.sh
+# Проверка: curl http://localhost:8080/health
 #
 # OOM: уменьшите CTX_SIZE (дефолт 50000), -ngl и/или cache types; при «каше» попробуйте bf16/f16 для KV (больше VRAM).
 # MoE: в .env CPU_MOE=1 при необходимости; число слоёв MoE на CPU — N_CPU_MOE (дефолт 99 в command); для dense оставьте CPU_MOE=0.
@@ -26,12 +18,7 @@ version: "3.8"
 
 services:
   llama-server:
-    # image: ghcr.io/ggml-org/llama.cpp:server-cuda13
+    image: ghcr.io/ggml-org/llama.cpp:server-cuda
-    image: llama-turboquant:${LLAMA_IMAGE_TAG:-server-cuda12}
-    build:
-      context: ${LLAMA_CPP_SRC:-../llama-cpp-turboquant}
-      # Relative to context (sibling of llama-space). Override for CUDA 13 — см. шапку.
-      dockerfile: ${LLAMA_DOCKERFILE:-../llama-space/docker/server-cuda12.Dockerfile}
     container_name: llama-server
     ports:
       # Явный IPv4 на хосте (избегаем привязки только к [::] в части окружений).
@@ -61,7 +48,7 @@ services:
       - "-c"
       - "${CTX_SIZE:-50000}"
       - "-np"
-      - "4"
+      - "1"
       - "-fa"
       - "on"
       - "--cache-type-k"

docker/build.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# Сборка образов llama-cpp-turboquant (CUDA 12 / CUDA 13).
-# Контекст — клон TheTom/llama-cpp-turboquant; Dockerfile'ы — в этом репозитории.
-#
-#   ./docker/build.sh          # CUDA 12 (RTX 30xx)
-#   ./docker/build.sh 13       # CUDA 13 / sm_120 (RTX 50xx)
-#   ./docker/build.sh 12 13    # оба
-#
-# Переменные: LLAMA_CPP_SRC, DOCKER (docker|podman)
-
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-SRC="${LLAMA_CPP_SRC:-$(cd "$ROOT/../llama-cpp-turboquant" && pwd)}"
-DOCKER="${DOCKER:-docker}"
-
-if [[ ! -f "$SRC/CMakeLists.txt" ]]; then
-  echo "llama.cpp source not found at: $SRC" >&2
-  echo "Set LLAMA_CPP_SRC to the TheTom/llama-cpp-turboquant checkout." >&2
-  exit 1
-fi
-
-build_one() {
-  local ver="$1"
-  local df tag
-  case "$ver" in
-    12)
-      df="$ROOT/docker/server-cuda12.Dockerfile"
-      tag="llama-turboquant:server-cuda12"
-      ;;
-    13)
-      df="$ROOT/docker/server-cuda13.Dockerfile"
-      tag="llama-turboquant:server-cuda13"
-      ;;
-    *)
-      echo "Unknown CUDA major: $ver (use 12 or 13)" >&2
-      exit 1
-      ;;
-  esac
-
-  echo "==> Building $tag"
-  echo "    Dockerfile: $df"
-  echo "    Context:    $SRC"
-  "$DOCKER" build -f "$df" -t "$tag" "$SRC"
-}
-
-if [[ $# -eq 0 ]]; then
-  set -- 12
-fi
-
-for ver in "$@"; do
-  build_one "$ver"
-done

docker/server-cuda12.Dockerfile

@@ -1,90 +0,0 @@
-# llama-cpp-turboquant → llama-server (CUDA 12).
-# Build context: root of https://github.com/TheTom/llama-cpp-turboquant
-#   docker build -f docker/server-cuda12.Dockerfile -t llama-turboquant:server-cuda12 ../llama-cpp-turboquant
-#
-# Для RTX 30xx (sm_86) и типичных драйверов с CUDA 12.x.
-# Архитектуры — дефолт CMake (включая 86-real); не трогаем, чтобы не ломать
-# CMAKE_CUDA_ARCHITECTURES со списком через «;» в shell RUN.
-
-ARG UBUNTU_VERSION=24.04
-ARG CUDA_VERSION=12.8.1
-ARG GCC_VERSION=14
-ARG BASE_CUDA_DEV_CONTAINER=docker.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
-ARG BASE_CUDA_RUN_CONTAINER=docker.io/nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
-
-ARG BUILD_DATE=N/A
-ARG APP_VERSION=N/A
-ARG APP_REVISION=N/A
-
-ARG NODE_VERSION=24
-
-FROM docker.io/node:$NODE_VERSION AS web
-
-ARG APP_VERSION
-
-WORKDIR /app/tools/ui
-
-COPY tools/ui/package.json tools/ui/package-lock.json ./
-RUN npm ci
-
-COPY tools/ui/ ./
-RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
-
-FROM ${BASE_CUDA_DEV_CONTAINER} AS build
-
-ARG GCC_VERSION
-ARG CUDA_DOCKER_ARCH=default
-
-RUN apt-get update && \
-    apt-get install -y gcc-${GCC_VERSION} g++-${GCC_VERSION} build-essential cmake python3 python3-pip git libssl-dev libgomp1
-
-ENV CC=gcc-${GCC_VERSION} CXX=g++-${GCC_VERSION} CUDAHOSTCXX=g++-${GCC_VERSION}
-
-WORKDIR /app
-
-COPY . .
-
-COPY --from=web /app/tools/ui/dist tools/ui/dist
-
-RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
-    export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
-    fi && \
-    cmake -B build -DGGML_NATIVE=OFF -DGGML_CUDA=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DLLAMA_BUILD_TESTS=OFF ${CMAKE_ARGS} -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined . && \
-    cmake --build build --config Release -j"$(nproc)" --target llama-server
-
-RUN mkdir -p /app/lib && \
-    find build -name "*.so*" -exec cp -P {} /app/lib \;
-
-RUN mkdir -p /app/full \
-    && cp build/bin/llama-server /app/full/
-
-FROM ${BASE_CUDA_RUN_CONTAINER} AS server
-
-ARG BUILD_DATE=N/A
-ARG APP_VERSION=N/A
-ARG APP_REVISION=N/A
-LABEL org.opencontainers.image.created=$BUILD_DATE \
-      org.opencontainers.image.version=$APP_VERSION \
-      org.opencontainers.image.revision=$APP_REVISION \
-      org.opencontainers.image.title="llama-cpp-turboquant" \
-      org.opencontainers.image.description="llama-server with TurboQuant KV (CUDA 12)" \
-      org.opencontainers.image.source="https://github.com/TheTom/llama-cpp-turboquant"
-
-RUN apt-get update \
-    && apt-get install -y libgomp1 curl \
-    && apt autoremove -y \
-    && apt clean -y \
-    && rm -rf /tmp/* /var/tmp/* \
-    && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
-    && find /var/cache -type f -delete
-
-ENV LLAMA_ARG_HOST=0.0.0.0
-
-COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama-server /app/
-
-WORKDIR /app
-
-HEALTHCHECK CMD [ "curl", "-f", "http://localhost:8080/health" ]
-
-ENTRYPOINT [ "/app/llama-server" ]

docker/server-cuda13.Dockerfile

@@ -1,90 +0,0 @@
-# llama-cpp-turboquant → llama-server (CUDA 13).
-# Build context: root of https://github.com/TheTom/llama-cpp-turboquant
-#   docker build -f docker/server-cuda13.Dockerfile -t llama-turboquant:server-cuda13 ../llama-cpp-turboquant
-#
-# Для Blackwell (RTX 50xx, sm_120). Toolkit 13.1.x — под драйвер с nvidia-smi CUDA 13.1.
-# Только 120a-real: быстрее сборка, FP4 tensor cores; не для карт старше Blackwell.
-
-ARG UBUNTU_VERSION=24.04
-ARG CUDA_VERSION=13.1.2
-ARG GCC_VERSION=14
-ARG BASE_CUDA_DEV_CONTAINER=docker.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
-ARG BASE_CUDA_RUN_CONTAINER=docker.io/nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
-
-ARG BUILD_DATE=N/A
-ARG APP_VERSION=N/A
-ARG APP_REVISION=N/A
-
-ARG NODE_VERSION=24
-
-FROM docker.io/node:$NODE_VERSION AS web
-
-ARG APP_VERSION
-
-WORKDIR /app/tools/ui
-
-COPY tools/ui/package.json tools/ui/package-lock.json ./
-RUN npm ci
-
-COPY tools/ui/ ./
-RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
-
-FROM ${BASE_CUDA_DEV_CONTAINER} AS build
-
-ARG GCC_VERSION
-# 120 → CMake подменит на 120a; явно 120a-real под Blackwell (не forwards-compatible).
-ARG CUDA_DOCKER_ARCH=120a-real
-
-RUN apt-get update && \
-    apt-get install -y gcc-${GCC_VERSION} g++-${GCC_VERSION} build-essential cmake python3 python3-pip git libssl-dev libgomp1
-
-ENV CC=gcc-${GCC_VERSION} CXX=g++-${GCC_VERSION} CUDAHOSTCXX=g++-${GCC_VERSION}
-
-WORKDIR /app
-
-COPY . .
-
-COPY --from=web /app/tools/ui/dist tools/ui/dist
-
-RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
-    export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
-    fi && \
-    cmake -B build -DGGML_NATIVE=OFF -DGGML_CUDA=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DLLAMA_BUILD_TESTS=OFF ${CMAKE_ARGS} -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined . && \
-    cmake --build build --config Release -j"$(nproc)" --target llama-server
-
-RUN mkdir -p /app/lib && \
-    find build -name "*.so*" -exec cp -P {} /app/lib \;
-
-RUN mkdir -p /app/full \
-    && cp build/bin/llama-server /app/full/
-
-FROM ${BASE_CUDA_RUN_CONTAINER} AS server
-
-ARG BUILD_DATE=N/A
-ARG APP_VERSION=N/A
-ARG APP_REVISION=N/A
-LABEL org.opencontainers.image.created=$BUILD_DATE \
-      org.opencontainers.image.version=$APP_VERSION \
-      org.opencontainers.image.revision=$APP_REVISION \
-      org.opencontainers.image.title="llama-cpp-turboquant" \
-      org.opencontainers.image.description="llama-server with TurboQuant KV (CUDA 13 / sm_120)" \
-      org.opencontainers.image.source="https://github.com/TheTom/llama-cpp-turboquant"
-
-RUN apt-get update \
-    && apt-get install -y libgomp1 curl \
-    && apt autoremove -y \
-    && apt clean -y \
-    && rm -rf /tmp/* /var/tmp/* \
-    && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
-    && find /var/cache -type f -delete
-
-ENV LLAMA_ARG_HOST=0.0.0.0
-
-COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama-server /app/
-
-WORKDIR /app
-
-HEALTHCHECK CMD [ "curl", "-f", "http://localhost:8080/health" ]
-
-ENTRYPOINT [ "/app/llama-server" ]

local-llama.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=llama-server (local-llama)
+After=network.target
+
+[Service]
+Type=simple
+User=bazzite
+Group=bazzite
+WorkingDirectory=/home/bazzite/llama-space
+EnvironmentFile=/home/bazzite/llama-space/.env
+ExecStart=/home/bazzite/llama-space/local-llama.sh
+Restart=on-failure
+RestartSec=5
+
+# CPU pinning: taskset -c 0-11
+CPUAffinity=0-11
+
+# ulimit -l unlimited → lock memory (mlockall / MADV_POPULATE_READ + mlock)
+LimitMEMLOCK=infinity
+
+# Не убивать OOM killer — модель критична
+OOMScoreAdjust=-900
+
+[Install]
+WantedBy=multi-user.target

local-llama.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export PATH=/usr/local/cuda-13.3/bin${PATH:+:${PATH}}
+export LD_LIBRARY_PATH=/usr/local/cuda-13.3/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+
+set -euo pipefail
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+/home/bazzite/Документы/llama-cpp-turboquant/build/bin/llama-server \
+  --models-preset "${DIR}/models.ini" \
+  --models-dir "${DIR}/models" \
+  -np 1

models.ini

@@ -0,0 +1,24 @@
+[DEFAULT]
+host = 0.0.0.0
+port = 8080
+flash-attn = on
+chat-template = jinja
+reasoning = on
+reasoning-budget = -1
+cache-type-k = turbo4
+cache-type-v = turbo3
+no-mmap = true
+mlock = true
+ctx-checkpoints = true
+cache-ram = -1
+batch-size = 2048
+ubatch-size = 2048
+
+[Qwen3.6-35B]
+model = models/Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
+n-gpu-layers = 99
+n-cpu-moe = 29
+ctx-size = 170000
+threads = 12
+spec-type = draft-mtp
+spec-draft-n-max = 2

podman-llama.sh

@@ -9,8 +9,6 @@
 # Переменные — как в compose / .env: PORT, MODEL_FILE, CPU_MOE, N_CPU_MOE, CTX_SIZE,
 #   CACHE_TYPE_K, CACHE_TYPE_V, THREADS; образ: IMAGE (по умолчанию server-cuda).
 # Порт на хосте: только IPv4 (BIND_HOST по умолчанию 0.0.0.0), иначе Podman часто вешает [::].
-# Монтирование models: по умолчанию :ro,z (SELinux — иначе часто Permission denied в rootless).
-#   Свой суффикс: PODMAN_MODELS_OPTS=z  или  PODMAN_MODELS_OPTS=Z  или пусто PODMAN_MODELS_OPTS=
 
 set -euo pipefail
 
@@ -32,9 +30,8 @@ fi
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=turbo2}"
 : "${THREADS:=8}"
-: "${IMAGE:=ghcr.io/ggml-org/llama.cpp:server-cuda}"
+: "${IMAGE:=ghcr.io/ggml-org/llama.cpp:server-cuda13}"
 : "${BIND_HOST:=0.0.0.0}"
-: "${PODMAN_MODELS_OPTS:=z}"
 
 # Разбить по пробелам для podman (пусто = только CDI-устройство).
 # Пример: PODMAN_GPU_FLAGS='--gpus all'
@@ -45,9 +42,10 @@ podman run -d \
   --replace \
   --restart no \
   --shm-size 1g \
+  --cpuset-cpus="0-11" \
   --ulimit memlock=-1:-1 \
   -p "${BIND_HOST}:${PORT}:8080" \
-  -v "${ROOT}/models:/models:ro${PODMAN_MODELS_OPTS:+,${PODMAN_MODELS_OPTS}}" \
+  -v "${ROOT}/models:/models:ro" \
   -e NVIDIA_VISIBLE_DEVICES=all \
   -e "LLAMA_ARG_CPU_MOE=${CPU_MOE}" \
   "${PODMAN_GPU_FLAGS_ARR[@]}" \
@@ -55,7 +53,9 @@ podman run -d \
   -m "/models/${MODEL_FILE}" \
   --host 0.0.0.0 \
   --port 8080 \
+  -ngl "${N_GPU_LAYERS}" \
   --n-cpu-moe "${N_CPU_MOE}" \
+  --spec-type draft-mtp --spec-draft-n-max 3 \
   -c "${CTX_SIZE}" \
   -np 1 \
   -fa on \
@@ -64,7 +64,7 @@ podman run -d \
   --no-mmap \
   --mlock \
   --ctx-checkpoints 1 \
-  --cache-ram 0 \
+  --cache-ram -1 \
   --jinja \
   --reasoning on \
   --reasoning-budget -1 \