Initial commit: Complete deployment scripts, power governor, systemd units, and architecture documentation for Turing multi-GPU LLM rig

2026-09-01 01:55:19 +00:00
commit aeb72cecfa
13 changed files with 1925 additions and 0 deletions
@@ -0,0 +1,12 @@
+[Unit]
+Description=Dynamic GPU Power & Clock Governor for CMP 50HX
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /opt/llama-server/power-governor.py
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,33 @@
+[Unit]
+Description=llama.cpp OpenAI-compatible server (Qwen3.8-27B Q4_K_P, 3x Turing GPUs, 256K ctx)
+Documentation=file:///opt/llama-server/README.md
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+# Required: without this the loader binds a stale libcuda.so.1 (550) from
+# /usr/lib/x86_64-linux-gnu/nvidia/current and CUDA init fails with
+# "system has unsupported display driver / cuda driver combination".
+Environment=LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
+ExecStart=/opt/llama-server/start.sh
+Restart=on-failure
+RestartSec=10
+# Model load takes ~20s; give it time to come up cleanly
+TimeoutStartSec=300
+TimeoutStopSec=60
+# Basic hardening (server only listens on :8080, reads models read-only)
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths=
+NoNewPrivileges=true
+PrivateTmp=true
+# Do not cap memory/CPU; inference needs all of it
+LimitNOFILE=1048576
+# NUMA Affinity: Pin to CPU Socket 1 (NUMA Node 1) where Riser 1 GPUs physically connect (24-thread v2 CPU)
+CPUAffinity=6-11,18-23
+NUMAPolicy=bind
+NUMAMask=1
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,19 @@
+[Unit]
+Description=Ollama+Anthropic API proxy backed by llama.cpp (Qwen3.8-27B)
+Documentation=file:///opt/llama-server/USAGE-CLIENTS.md
+After=network-online.target llama-server.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /opt/llama-server/ollama-proxy.py
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+# proxy does not need GPU access itself; it forwards to llama-server on 127.0.0.1:8080
+NoNewPrivileges=true
+PrivateTmp=true
+LimitNOFILE=1048576
+
+[Install]
+WantedBy=multi-user.target