refactor: remove container-side power governor in favor of host-level cmp-tune
This commit is contained in:
wmantly
@@ -36,18 +36,17 @@ Total VRAM: 32 GB GDDR6 across 3 GPUs (Shared PCIe Root Complex)
|
||||
├── scripts/
|
||||
│ ├── build-nccl-llama.sh # Compiles llama.cpp with CUDA 12 + NCCL + sm_75 optimizations
|
||||
│ ├── ollama-proxy.py # High-performance Ollama/OpenAI/Anthropic proxy with thinking control
|
||||
│ ├── power-governor.py # Dynamic GPU power & clock governor for CMP 50HX cards
|
||||
│ ├── start-server.sh # Production start script for llama-server
|
||||
│ └── tune-gpus.sh # Helper script for power limits & persistence mode
|
||||
│ └── start-server.sh # Production start script for llama-server
|
||||
├── systemd/
|
||||
│ ├── llama-server.service # Systemd unit with NUMA socket pinning
|
||||
│ ├── ollama-proxy.service # Systemd unit for API proxy
|
||||
│ └── gpu-power-governor.service # Systemd unit for automated power governor
|
||||
│ └── ollama-proxy.service # Systemd unit for API proxy
|
||||
├── tuning/
|
||||
│ └── cmp-tune.conf # Host-level CMP 50HX tuning profiles (stable-fast, efficient, insane)
|
||||
├── vbios/
|
||||
│ └── msi_cmp50hx_90.02.60.00.17.rom # Tuned MSI VBIOS ROM (31W idle & 25% min fan curve)
|
||||
└── docs/
|
||||
├── SETUP_GUIDE.md # Complete step-by-step setup guide on any Linux server
|
||||
├── PROXMOX_LXC_GUIDE.md # GPU passthrough & NVML permissions in Proxmox LXC
|
||||
├── PROXMOX_LXC_GUIDE.md # GPU passthrough, CPU socket pinning & host power management
|
||||
├── HARDWARE_LEARNINGS.md # Deep dive: CMP VBIOS, PCIe Gen1 vs Gen3, riser cables, 20GB mods
|
||||
├── 20GB_MOD_GUIDE.md # Hardware strapping & driver patching for 20GB TU102 mods
|
||||
├── CMP_ECOSYSTEM_AND_COMMUNITY.md # Master ecosystem guide: projects, techniques & community
|
||||
@@ -67,7 +66,7 @@ bash scripts/build-nccl-llama.sh /opt/llama.cpp
|
||||
```bash
|
||||
sudo cp systemd/*.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now llama-server.service ollama-proxy.service gpu-power-governor.service
|
||||
sudo systemctl enable --now llama-server.service ollama-proxy.service
|
||||
```
|
||||
|
||||
### 3. Connect from OpenWebUI
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
power-governor.py — Resilient Dynamic GPU Power Governor for CMP 50HX.
|
||||
|
||||
Tracks llama-server slot state + GPU utilization with fail-safe active retention:
|
||||
- Never downclocks while llama-server is processing a request or busy in CUDA.
|
||||
- Holds full 1900 MHz boost clocks for 45s of silence before entering low-power idle.
|
||||
- Safe 150W power cap enforced on boot.
|
||||
"""
|
||||
|
||||
import time
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import json
|
||||
import sys
|
||||
|
||||
CMP_GPUS = "1,2"
|
||||
IDLE_TIMEOUT_SEC = 45
|
||||
LOW_POWER_CLOCK = 300
|
||||
|
||||
def run_cmd(*args):
|
||||
try:
|
||||
subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_gpu_utilization():
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["nvidia-smi", f"-i={CMP_GPUS}", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"],
|
||||
universal_newlines=True,
|
||||
timeout=2.0
|
||||
)
|
||||
utils = [int(line.strip()) for line in out.strip().split("\n") if line.strip().isdigit()]
|
||||
return max(utils) if utils else 0
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def is_server_processing():
|
||||
try:
|
||||
req = urllib.request.Request("http://127.0.0.1:8080/slots", headers={"User-Agent": "power-governor"})
|
||||
with urllib.request.urlopen(req, timeout=4.0) as r:
|
||||
slots = json.loads(r.read().decode())
|
||||
return any(bool(s.get("is_processing")) for s in slots)
|
||||
except Exception:
|
||||
# If server is busy or timing out, assume it is ACTIVE to prevent downclocking
|
||||
return True
|
||||
|
||||
def main():
|
||||
print("[power-governor] Initializing GPU persistence and 150W power cap...", flush=True)
|
||||
run_cmd("nvidia-smi", "-pm", "1")
|
||||
run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-pl", "150")
|
||||
run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
|
||||
|
||||
is_low_power = False
|
||||
last_active = time.time()
|
||||
|
||||
print("[power-governor] Monitoring llama-server state & GPU compute...", flush=True)
|
||||
while True:
|
||||
try:
|
||||
active = is_server_processing()
|
||||
if not active:
|
||||
# Double-check GPU compute utilization before declaring idle
|
||||
if get_gpu_utilization() > 0:
|
||||
active = True
|
||||
|
||||
now = time.time()
|
||||
|
||||
if active:
|
||||
last_active = now
|
||||
if is_low_power:
|
||||
run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:07:00.0", "--pstate", "16")
|
||||
run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:21:00.0", "--pstate", "16")
|
||||
run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
|
||||
is_low_power = False
|
||||
print(f"[{time.strftime('%X')}] Inference Active -> Boost clocks engaged (P0/P16, 1900 MHz)", flush=True)
|
||||
else:
|
||||
if not is_low_power and (now - last_active) > IDLE_TIMEOUT_SEC:
|
||||
run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:07:00.0", "--pstate", "8")
|
||||
run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:21:00.0", "--pstate", "8")
|
||||
run_cmd("nvidia-smi", f"-i={CMP_GPUS}", f"-lgc={LOW_POWER_CLOCK},{LOW_POWER_CLOCK}")
|
||||
is_low_power = True
|
||||
print(f"[{time.strftime('%X')}] Inference Idle (> {IDLE_TIMEOUT_SEC}s) -> Deep low-power state engaged (NVAPI P8 + 300 MHz)", flush=True)
|
||||
|
||||
time.sleep(0.5 if is_low_power else 1.5)
|
||||
except Exception as e:
|
||||
time.sleep(2.0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ==============================================================================
|
||||
# GPU Persistence Mode and Power Cap Tuner
|
||||
# Sets safe 150W power cap on CMP 50HX cards and activates persistence mode.
|
||||
# ==============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
echo "==> Enabling NVIDIA Persistence Mode..."
|
||||
nvidia-smi -pm 1
|
||||
|
||||
echo "==> Setting 150W Power Cap on CMP 50HX GPUs (index 1 and 2)..."
|
||||
nvidia-smi -i 1,2 -pl 150
|
||||
|
||||
echo "==> Resetting GPU clocks to unconstrained boost..."
|
||||
nvidia-smi -rgc
|
||||
|
||||
echo "==> Current GPU States:"
|
||||
nvidia-smi --query-gpu=index,name,power.draw,power.limit,clocks.gr,clocks.mem --format=csv
|
||||
@@ -1,12 +0,0 @@
|
||||
[Unit]
|
||||
Description=Dynamic GPU Power & Clock Governor for CMP 50HX
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/python3 /opt/llama-server/power-governor.py
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue
Block a user