Initial commit: Complete deployment scripts, power governor, systemd units, and architecture documentation for Turing multi-GPU LLM rig

2026-09-01 01:55:19 +00:00
commit aeb72cecfa
13 changed files with 1925 additions and 0 deletions
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# Production startup script for llama-server with NCCL tensor parallelism
+# Supports: 256K Context Window, Multimodal Vision Projector, Flash Attention,
+#           q4_0 KV Cache quantization, and Jinja reasoning control.
+# ==============================================================================
+set -euo pipefail
+
+MODEL_PATH="${MODEL_PATH:-/opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf}"
+MMPROJ_PATH="${MMPROJ_PATH:-/opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}"
+BINARY="${LLAMA_BINARY:-/opt/llama.cpp/build-nccl/bin/llama-server}"
+
+export LD_LIBRARY_PATH="/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
+
+exec "$BINARY" \
+  -m "$MODEL_PATH" \
+  --mmproj "$MMPROJ_PATH" \
+  -ngl 99 \
+  -c 262144 \
+  --parallel 1 \
+  --image-max-tokens 2048 \
+  --split-mode tensor \
+  --flash-attn on \
+  --batch-size 1024 \
+  --ubatch-size 256 \
+  --jinja \
+  --threads 12 \
+  --cache-type-k q4_0 \
+  --cache-type-v q4_0 \
+  --host 0.0.0.0 \
+  --port 8080