32 lines
1.1 KiB
Bash
Executable File
32 lines
1.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ==============================================================================
|
|
# Production startup script for llama-server with NCCL tensor parallelism
|
|
# Supports: 256K Context Window, Multimodal Vision Projector, Flash Attention,
|
|
# q4_0 KV Cache quantization, and Jinja reasoning control.
|
|
# ==============================================================================
|
|
set -euo pipefail
|
|
|
|
MODEL_PATH="${MODEL_PATH:-/opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf}"
|
|
MMPROJ_PATH="${MMPROJ_PATH:-/opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}"
|
|
BINARY="${LLAMA_BINARY:-/opt/llama.cpp/build-nccl/bin/llama-server}"
|
|
|
|
export LD_LIBRARY_PATH="/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
|
|
|
|
exec "$BINARY" \
|
|
-m "$MODEL_PATH" \
|
|
--mmproj "$MMPROJ_PATH" \
|
|
-ngl 99 \
|
|
-c 262144 \
|
|
--parallel 1 \
|
|
--image-max-tokens 2048 \
|
|
--split-mode tensor \
|
|
--flash-attn on \
|
|
--batch-size 1024 \
|
|
--ubatch-size 256 \
|
|
--jinja \
|
|
--threads 12 \
|
|
--cache-type-k q4_0 \
|
|
--cache-type-v q4_0 \
|
|
--host 0.0.0.0 \
|
|
--port 8080
|