Initial commit: Complete deployment scripts, power governor, systemd units, and architecture documentation for Turing multi-GPU LLM rig

scripts/build-nccl-llama.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# Build script for compiling llama.cpp with NVIDIA NCCL tensor parallelism
+# Optimized for Turing GPUs (RTX 2060, CMP 50HX, sm_75)
+# ==============================================================================
+set -euo pipefail
+
+LLAMA_DIR="${1:-/opt/llama.cpp}"
+BUILD_DIR="${LLAMA_DIR}/build-nccl"
+
+echo "==> Ensuring build dependencies are installed..."
+apt-get update -qq && apt-get install -y -qq \
+    build-essential cmake git ninja-build \
+    libcurl4-openssl-dev libssl-dev pkg-config
+
+echo "==> Configuring CMake for llama.cpp with NCCL and CUDA sm_75..."
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+cmake .. \
+    -GNinja \
+    -DGGML_CUDA=ON \
+    -DGGML_CUDA_GRAPHS=ON \
+    -DGGML_CUDA_FORCE_CUBLAS=ON \
+    -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 \
+    -DGGML_CUDA_ARCHITECTURES="75" \
+    -DCMAKE_BUILD_TYPE=Release
+
+echo "==> Building llama-server..."
+ninja llama-server
+
+echo "==> Build complete! Binary located at: ${BUILD_DIR}/bin/llama-server"

scripts/power-governor.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+power-governor.py — Dynamic clock and power governor for CMP 50HX GPUs.
+
+Directly queries llama-server slot state for 100% accurate active/idle tracking:
+- When generating (is_processing: True): Instantly sets full 1900 MHz boost clocks.
+- When idle (is_processing: False for > 15s): Downclocks CMP cards to 600 MHz (~32W/card).
+"""
+
+import time
+import subprocess
+import urllib.request
+import json
+import sys
+
+CMP_GPUS = "1,2"
+IDLE_TIMEOUT_SEC = 15
+LOW_POWER_CLOCK = 300
+
+def run_cmd(*args):
+    try:
+        subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
+    except Exception:
+        pass
+
+def is_server_processing():
+    try:
+        req = urllib.request.Request("http://127.0.0.1:8080/slots", headers={"User-Agent": "power-governor"})
+        with urllib.request.urlopen(req, timeout=1.0) as r:
+            slots = json.loads(r.read().decode())
+            return any(bool(s.get("is_processing")) for s in slots)
+    except Exception:
+        return False
+
+def main():
+    print("[power-governor] Initializing GPU persistence and 150W power cap...", flush=True)
+    run_cmd("nvidia-smi", "-pm", "1")
+    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-pl", "150")
+    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
+
+    is_low_power = False
+    last_active = time.time()
+
+    print("[power-governor] Monitoring llama-server processing state...", flush=True)
+    while True:
+        try:
+            active = is_server_processing()
+            now = time.time()
+
+            if active:
+                last_active = now
+                if is_low_power:
+                    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
+                    is_low_power = False
+                    print(f"[{time.strftime('%X')}] Inference Active -> Boost clocks engaged (1900 MHz)", flush=True)
+            else:
+                if not is_low_power and (now - last_active) > IDLE_TIMEOUT_SEC:
+                    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", f"-lgc={LOW_POWER_CLOCK},{LOW_POWER_CLOCK}")
+                    is_low_power = True
+                    print(f"[{time.strftime('%X')}] Inference Idle -> Low-power state engaged (600 MHz, ~32W/card)", flush=True)
+
+            time.sleep(0.5 if is_low_power else 1.0)
+        except Exception as e:
+            time.sleep(3.0)
+
+if __name__ == "__main__":
+    main()

scripts/start-server.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# Production startup script for llama-server with NCCL tensor parallelism
+# Supports: 256K Context Window, Multimodal Vision Projector, Flash Attention,
+#           q4_0 KV Cache quantization, and Jinja reasoning control.
+# ==============================================================================
+set -euo pipefail
+
+MODEL_PATH="${MODEL_PATH:-/opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf}"
+MMPROJ_PATH="${MMPROJ_PATH:-/opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}"
+BINARY="${LLAMA_BINARY:-/opt/llama.cpp/build-nccl/bin/llama-server}"
+
+export LD_LIBRARY_PATH="/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
+
+exec "$BINARY" \
+  -m "$MODEL_PATH" \
+  --mmproj "$MMPROJ_PATH" \
+  -ngl 99 \
+  -c 262144 \
+  --parallel 1 \
+  --image-max-tokens 2048 \
+  --split-mode tensor \
+  --flash-attn on \
+  --batch-size 1024 \
+  --ubatch-size 256 \
+  --jinja \
+  --threads 12 \
+  --cache-type-k q4_0 \
+  --cache-type-v q4_0 \
+  --host 0.0.0.0 \
+  --port 8080

scripts/tune-gpus.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# GPU Persistence Mode and Power Cap Tuner
+# Sets safe 150W power cap on CMP 50HX cards and activates persistence mode.
+# ==============================================================================
+set -euo pipefail
+
+echo "==> Enabling NVIDIA Persistence Mode..."
+nvidia-smi -pm 1
+
+echo "==> Setting 150W Power Cap on CMP 50HX GPUs (index 1 and 2)..."
+nvidia-smi -i 1,2 -pl 150
+
+echo "==> Resetting GPU clocks to unconstrained boost..."
+nvidia-smi -rgc
+
+echo "==> Current GPU States:"
+nvidia-smi --query-gpu=index,name,power.draw,power.limit,clocks.gr,clocks.mem --format=csv