refactor: remove container-side power governor in favor of host-level cmp-tune

README.md

@@ -36,18 +36,17 @@ Total VRAM:  32 GB GDDR6 across 3 GPUs (Shared PCIe Root Complex)
 ├── scripts/
 │   ├── build-nccl-llama.sh             # Compiles llama.cpp with CUDA 12 + NCCL + sm_75 optimizations
 │   ├── ollama-proxy.py                 # High-performance Ollama/OpenAI/Anthropic proxy with thinking control
-│   ├── power-governor.py               # Dynamic GPU power & clock governor for CMP 50HX cards
+│   └── start-server.sh                 # Production start script for llama-server
-│   ├── start-server.sh                 # Production start script for llama-server
-│   └── tune-gpus.sh                    # Helper script for power limits & persistence mode
 ├── systemd/
 │   ├── llama-server.service            # Systemd unit with NUMA socket pinning
-│   ├── ollama-proxy.service            # Systemd unit for API proxy
+│   └── ollama-proxy.service            # Systemd unit for API proxy
-│   └── gpu-power-governor.service      # Systemd unit for automated power governor
+├── tuning/
+│   └── cmp-tune.conf                   # Host-level CMP 50HX tuning profiles (stable-fast, efficient, insane)
 ├── vbios/
 │   └── msi_cmp50hx_90.02.60.00.17.rom  # Tuned MSI VBIOS ROM (31W idle & 25% min fan curve)
 └── docs/
     ├── SETUP_GUIDE.md                  # Complete step-by-step setup guide on any Linux server
-    ├── PROXMOX_LXC_GUIDE.md            # GPU passthrough & NVML permissions in Proxmox LXC
+    ├── PROXMOX_LXC_GUIDE.md            # GPU passthrough, CPU socket pinning & host power management
     ├── HARDWARE_LEARNINGS.md           # Deep dive: CMP VBIOS, PCIe Gen1 vs Gen3, riser cables, 20GB mods
     ├── 20GB_MOD_GUIDE.md               # Hardware strapping & driver patching for 20GB TU102 mods
     ├── CMP_ECOSYSTEM_AND_COMMUNITY.md  # Master ecosystem guide: projects, techniques & community
@@ -67,7 +66,7 @@ bash scripts/build-nccl-llama.sh /opt/llama.cpp
 ```bash
 sudo cp systemd/*.service /etc/systemd/system/
 sudo systemctl daemon-reload
-sudo systemctl enable --now llama-server.service ollama-proxy.service gpu-power-governor.service
+sudo systemctl enable --now llama-server.service ollama-proxy.service
 ```
 
 ### 3. Connect from OpenWebUI

scripts/power-governor.py

@@ -1,90 +0,0 @@
-#!/usr/bin/env python3
-"""
-power-governor.py — Resilient Dynamic GPU Power Governor for CMP 50HX.
-
-Tracks llama-server slot state + GPU utilization with fail-safe active retention:
-- Never downclocks while llama-server is processing a request or busy in CUDA.
-- Holds full 1900 MHz boost clocks for 45s of silence before entering low-power idle.
-- Safe 150W power cap enforced on boot.
-"""
-
-import time
-import subprocess
-import urllib.request
-import json
-import sys
-
-CMP_GPUS = "1,2"
-IDLE_TIMEOUT_SEC = 45
-LOW_POWER_CLOCK = 300
-
-def run_cmd(*args):
-    try:
-        subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
-    except Exception:
-        pass
-
-def get_gpu_utilization():
-    try:
-        out = subprocess.check_output(
-            ["nvidia-smi", f"-i={CMP_GPUS}", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"],
-            universal_newlines=True,
-            timeout=2.0
-        )
-        utils = [int(line.strip()) for line in out.strip().split("\n") if line.strip().isdigit()]
-        return max(utils) if utils else 0
-    except Exception:
-        return 0
-
-def is_server_processing():
-    try:
-        req = urllib.request.Request("http://127.0.0.1:8080/slots", headers={"User-Agent": "power-governor"})
-        with urllib.request.urlopen(req, timeout=4.0) as r:
-            slots = json.loads(r.read().decode())
-            return any(bool(s.get("is_processing")) for s in slots)
-    except Exception:
-        # If server is busy or timing out, assume it is ACTIVE to prevent downclocking
-        return True
-
-def main():
-    print("[power-governor] Initializing GPU persistence and 150W power cap...", flush=True)
-    run_cmd("nvidia-smi", "-pm", "1")
-    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-pl", "150")
-    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
-
-    is_low_power = False
-    last_active = time.time()
-
-    print("[power-governor] Monitoring llama-server state & GPU compute...", flush=True)
-    while True:
-        try:
-            active = is_server_processing()
-            if not active:
-                # Double-check GPU compute utilization before declaring idle
-                if get_gpu_utilization() > 0:
-                    active = True
-
-            now = time.time()
-
-            if active:
-                last_active = now
-                if is_low_power:
-                    run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:07:00.0", "--pstate", "16")
-                    run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:21:00.0", "--pstate", "16")
-                    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", "-rgc")
-                    is_low_power = False
-                    print(f"[{time.strftime('%X')}] Inference Active -> Boost clocks engaged (P0/P16, 1900 MHz)", flush=True)
-            else:
-                if not is_low_power and (now - last_active) > IDLE_TIMEOUT_SEC:
-                    run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:07:00.0", "--pstate", "8")
-                    run_cmd("python3", "/opt/turing-multi-gpu-llm-server/tools/cmp-pstate.py", "--bus-id", "00000000:21:00.0", "--pstate", "8")
-                    run_cmd("nvidia-smi", f"-i={CMP_GPUS}", f"-lgc={LOW_POWER_CLOCK},{LOW_POWER_CLOCK}")
-                    is_low_power = True
-                    print(f"[{time.strftime('%X')}] Inference Idle (> {IDLE_TIMEOUT_SEC}s) -> Deep low-power state engaged (NVAPI P8 + 300 MHz)", flush=True)
-
-            time.sleep(0.5 if is_low_power else 1.5)
-        except Exception as e:
-            time.sleep(2.0)
-
-if __name__ == "__main__":
-    main()

scripts/tune-gpus.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# ==============================================================================
-# GPU Persistence Mode and Power Cap Tuner
-# Sets safe 150W power cap on CMP 50HX cards and activates persistence mode.
-# ==============================================================================
-set -euo pipefail
-
-echo "==> Enabling NVIDIA Persistence Mode..."
-nvidia-smi -pm 1
-
-echo "==> Setting 150W Power Cap on CMP 50HX GPUs (index 1 and 2)..."
-nvidia-smi -i 1,2 -pl 150
-
-echo "==> Resetting GPU clocks to unconstrained boost..."
-nvidia-smi -rgc
-
-echo "==> Current GPU States:"
-nvidia-smi --query-gpu=index,name,power.draw,power.limit,clocks.gr,clocks.mem --format=csv

systemd/gpu-power-governor.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Dynamic GPU Power & Clock Governor for CMP 50HX
-After=network.target
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/python3 /opt/llama-server/power-governor.py
-Restart=always
-RestartSec=5
-
-[Install]
-WantedBy=multi-user.target