33 lines
1.1 KiB
Bash
Executable File
33 lines
1.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
|
|
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
|
|
export LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
|
|
|
|
# Low-Latency NCCL & CUDA Driver Pipeline Optimizations
|
|
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
|
export NCCL_BUFFSIZE=2097152
|
|
export NCCL_NET_GDR_LEVEL=0
|
|
export NCCL_P2P_DISABLE=0
|
|
export NCCL_ALGO=RING
|
|
export NCCL_PROTO=SIMPLE
|
|
|
|
exec /opt/llama.cpp/build-nccl/bin/llama-server \
|
|
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
|
|
--spec-type ngram-simple \
|
|
--spec-ngram-simple-size-n 24 \
|
|
--spec-ngram-simple-size-m 4 \
|
|
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
|
|
-ngl 99 \
|
|
-c 262144 \
|
|
--parallel 1 \
|
|
--image-max-tokens 2048 \
|
|
--split-mode tensor \
|
|
--flash-attn on \
|
|
--batch-size 1024 \
|
|
--ubatch-size 512 \
|
|
--jinja \
|
|
--threads 12 \
|
|
--cache-type-k q4_0 \
|
|
--cache-type-v q4_0 \
|
|
--host 0.0.0.0 \
|
|
--port 8080 |