wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: re-enable multimodal vision ViT projector offloaded to 12GB RTX 2060 (CUDA2)

Browse Source
This commit is contained in:
wmantly
2026-09-04 20:29:40 +00:00
parent 9ab0583cab
commit 41265cf9db
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-server.sh
+3
View File
@@ -13,6 +13,9 @@ export NCCL_PROTO=SIMPLE
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
--mmproj-device CUDA2 \
--image-min-tokens 1024 \
--spec-type draft-mtp \ --spec-type draft-mtp \
--spec-draft-n-max 2 \ --spec-draft-n-max 2 \
--numa split \ --numa split \