wmantly/turing-multi-gpu-llm-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(config): switch model to Qwen3.8-27B Q4_K_P with FastMTP 32K draft

Browse Source
This commit is contained in:
wmantly committed 2026-09-22 01:22:46 +00:00
1 parent 7269a96313
commit 1e98eea3bd
2 files changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/ollama-proxy.py
+3 -3
View File
@@ -353,7 +353,7 @@ def _model_entry(tag):
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q5_K_P", "quantization_level": "Q4_K_P",
"context_length": ctx, "context_length": ctx,
}, },
"capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"], "capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"],
@@ -376,7 +376,7 @@ def ollama_ps():
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q5_K_P", "quantization_level": "Q4_K_P",
"context_length": ctx, "context_length": ctx,
}, },
"expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z", "expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z",
@@ -401,7 +401,7 @@ def ollama_show(payload=None):
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q5_K_P", "quantization_level": "Q4_K_P",
"context_length": ctx, "context_length": ctx,
}, },
"model_info": { "model_info": {
scripts/start-server.sh
+1 -1
View File
@@ -17,7 +17,7 @@ RTX_DEV=$(/opt/llama.cpp-xrip/build-nccl/bin/llama-cli --list-devices | grep -i
RTX_DEV=${RTX_DEV:-CUDA0} RTX_DEV=${RTX_DEV:-CUDA0}
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
--spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \ --spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
--spec-draft-device "$RTX_DEV" \ --spec-draft-device "$RTX_DEV" \
--spec-draft-ngl all \ --spec-draft-ngl all \