diff --git a/scripts/ollama-proxy.py b/scripts/ollama-proxy.py index 0e7d947..b865cae 100644 --- a/scripts/ollama-proxy.py +++ b/scripts/ollama-proxy.py @@ -143,6 +143,10 @@ MODEL_TAGS = [ "Qwen3.8-27B-Uncensored", "qwen3.8-27b:latest", "qwen3.8-27b", + "qwen3.8-27b-aggressive-q3", + "qwen3.8-27b-aggressive-q3:latest", + "qwen3.8-27b-aggressive", + "qwen3.8-27b-aggressive:latest", ] # Set to False to hide reasoning_content / thinking blocks entirely. diff --git a/scripts/start-server.sh b/scripts/start-server.sh index 419eee8..ac154fd 100755 --- a/scripts/start-server.sh +++ b/scripts/start-server.sh @@ -1,6 +1,7 @@ #!/bin/bash # llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB) # Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism. +# Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters. export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu # Low-Latency NCCL & CUDA Driver Pipeline Optimizations @@ -13,14 +14,28 @@ export NCCL_PROTO=SIMPLE exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ + --spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \ + --spec-draft-device CUDA2 \ + --spec-draft-ngl all \ + --spec-type draft-mtp \ + --spec-draft-n-max 3 \ + --spec-draft-p-min 0 \ --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \ --mmproj-device CUDA2 \ --image-min-tokens 1024 \ - --spec-type draft-mtp \ - --spec-draft-n-max 2 \ + --temp 1.0 \ + --top-k 20 \ + --top-p 0.95 \ + --min-p 0 \ + --presence-penalty 0 \ + --repeat-penalty 1.0 \ + --reasoning on \ + --reasoning-effort xhigh \ + --reasoning-preserve \ + --reasoning-format deepseek \ --numa split \ -ngl 99 \ - -c 262144 \ + -c 204800 \ --parallel 1 \ --slot-save-path /var/cache/llama-slots \ --cache-ram 16384 \