From 2b4e826437b5d46ccb895cebaa6ea064d5c2d768 Mon Sep 17 00:00:00 2001 From: wmantly Date: Tue, 1 Sep 2026 14:55:56 +0000 Subject: [PATCH] Add sglang-inspired low-latency NCCL ring buffer and CUDA connection optimization flags to start-server.sh --- scripts/start-server.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/scripts/start-server.sh b/scripts/start-server.sh index fcf8eab..7f0d36b 100755 --- a/scripts/start-server.sh +++ b/scripts/start-server.sh @@ -1,20 +1,19 @@ -#!/usr/bin/env bash -# ============================================================================== -# Production startup script for llama-server with NCCL tensor parallelism -# Supports: 256K Context Window, Multimodal Vision Projector, Flash Attention, -# q4_0 KV Cache quantization, and Jinja reasoning control. -# ============================================================================== -set -euo pipefail +#!/bin/bash +# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB) +# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism. +export LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu -MODEL_PATH="${MODEL_PATH:-/opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf}" -MMPROJ_PATH="${MMPROJ_PATH:-/opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}" -BINARY="${LLAMA_BINARY:-/opt/llama.cpp/build-nccl/bin/llama-server}" +# Low-Latency NCCL & CUDA Driver Pipeline Optimizations +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NCCL_BUFFSIZE=2097152 +export NCCL_NET_GDR_LEVEL=0 +export NCCL_P2P_DISABLE=0 +export NCCL_ALGO=RING +export NCCL_PROTO=SIMPLE -export LD_LIBRARY_PATH="/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" - -exec "$BINARY" \ - -m "$MODEL_PATH" \ - --mmproj "$MMPROJ_PATH" \ +exec /opt/llama.cpp/build-nccl/bin/llama-server \ + -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ + --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \ -ngl 99 \ -c 262144 \ --parallel 1 \ @@ -28,4 +27,4 @@ exec "$BINARY" \ --cache-type-k q4_0 \ --cache-type-v q4_0 \ --host 0.0.0.0 \ - --port 8080 + --port 8080 \ No newline at end of file