#!/bin/bash
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
export LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu

# Low-Latency NCCL & CUDA Driver Pipeline Optimizations
export CUDA_DEVICE_MAX_CONNECTIONS=1
export NCCL_BUFFSIZE=2097152
export NCCL_NET_GDR_LEVEL=0
export NCCL_P2P_DISABLE=0
export NCCL_ALGO=RING
export NCCL_PROTO=SIMPLE

exec /opt/llama.cpp/build-nccl/bin/llama-server \
  -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
  --spec-type draft-mtp \
  --spec-draft-n-max 2 \
  -ngl 99 \
  -c 262144 \
  --parallel 1 \
  --split-mode tensor \
  --flash-attn on \
  --batch-size 1024 \
  --ubatch-size 512 \
  --jinja \
  --threads 12 \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --host 0.0.0.0 \
  --port 8080