#!/bin/bash
# llama-server launcher for Qwen3.8-27B on Turing GPUs (RTX 2060 12GB + CMP 50HX 10GB)
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
# Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters.
export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu

# Low-Latency NCCL & CUDA Driver Pipeline Optimizations
export CUDA_DEVICE_MAX_CONNECTIONS=1
export NCCL_BUFFSIZE=2097152
export NCCL_NET_GDR_LEVEL=0
export NCCL_P2P_DISABLE=0
export NCCL_ALGO=RING
export NCCL_PROTO=SIMPLE

# Dynamically target the 12GB RTX 2060 for mmproj and FastMTP draft model
RTX_DEV=$(/opt/llama.cpp-xrip/build-nccl/bin/llama-cli --list-devices | grep -i "RTX 2060" | awk '{print $1}' | tr -d ':')
RTX_DEV=${RTX_DEV:-CUDA0}

exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
  -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf \
  --spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
  --spec-draft-device "$RTX_DEV" \
  --spec-draft-ngl all \
  --spec-type draft-mtp \
  --spec-draft-n-max 3 \
  --spec-draft-p-min 0 \
  --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
  --mmproj-device "$RTX_DEV" \
  --image-min-tokens 1024 \
  --temp 1.0 \
  --top-k 20 \
  --top-p 0.95 \
  --min-p 0 \
  --presence-penalty 0 \
  --repeat-penalty 1.0 \
  --reasoning on \
  --reasoning-effort xhigh \
  --reasoning-preserve \
  --reasoning-format deepseek \
  --numa isolate \
  -ngl 99 \
  -c 131072 \
  --parallel 1 \
  --cache-prompt \
  --cache-reuse 64 \
  --cache-idle-slots \
  --slot-prompt-similarity 0.10 \
  --cache-ram 32768 \
  --slot-save-path /var/cache/llama-slots \
  --split-mode tensor \
  --flash-attn on \
  --batch-size 2048 \
  --ubatch-size 1024 \
  --jinja \
  --threads 12 \
  --cache-type-k q5_0 \
  --cache-type-v q5_0 \
  --host 0.0.0.0 \
  --port 8080