migrate to router mode: use models.ini for multi-model support

2026-06-22 17:23:29 +07:00
parent ed5a4fdbbc
commit b14fc46642
2 changed files with 29 additions and 21 deletions
@@ -0,0 +1,24 @@
+[DEFAULT]
+host = 0.0.0.0
+port = 8080
+flash-attn = on
+chat-template = jinja
+reasoning = on
+reasoning-budget = -1
+cache-type-k = turbo4
+cache-type-v = turbo3
+no-mmap = true
+mlock = true
+ctx-checkpoints = true
+cache-ram = -1
+batch-size = 2048
+ubatch-size = 2048
+
+[Qwen3.6-35B]
+model = models/Qwen3.6-35B-A3B-MTP-MXFP4_MOE.gguf
+n-gpu-layers = 99
+n-cpu-moe = 29
+ctx-size = 170000
+threads = 12
+spec-type = draft-mtp
+spec-draft-n-max = 2