Fix power governor active retention during long prompt evaluations and remove redundant proxy governor thread

scripts/ollama-proxy.py

@@ -26,54 +26,6 @@ import threading
 import re
 import subprocess
 
-# ----------------------------------------------------------------------------
-# GPU Power & Clock Governor (Automatic idle power reduction for CMP 50HX)
-# ----------------------------------------------------------------------------
-class GPUGovernor:
-    """Manages GPU clocks dynamically: drops CMP cards to low-power state when idle (saves ~90W),
-    and instantly unconstrains to full boost clocks during inference."""
-    def __init__(self, idle_timeout=30):
-        self.idle_timeout = idle_timeout
-        self.last_active = time.time()
-        self.is_low_power = False
-        self.lock = threading.Lock()
-        self._init_limits()
-        t = threading.Thread(target=self._monitor_loop, daemon=True)
-        t.start()
-
-    def _run_cmd(self, *args):
-        try:
-            subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
-        except Exception:
-            pass
-
-    def _init_limits(self):
-        # Ensure persistence mode and 150W safe power cap
-        self._run_cmd("nvidia-smi", "-pm", "1")
-        self._run_cmd("nvidia-smi", "-i", "1,2", "-pl", "150")
-
-    def wake(self):
-        with self.lock:
-            self.last_active = time.time()
-            if self.is_low_power:
-                self._run_cmd("nvidia-smi", "-rgc")
-                self.is_low_power = False
-
-    def touch(self):
-        with self.lock:
-            self.last_active = time.time()
-
-    def _monitor_loop(self):
-        while True:
-            time.sleep(5)
-            with self.lock:
-                if not self.is_low_power and (time.time() - self.last_active) > self.idle_timeout:
-                    # Drop CMP cards to low power idle clock
-                    self._run_cmd("nvidia-smi", "-i", "1,2", "-lgc", "600,600")
-                    self.is_low_power = True
-
-GOVERNOR = GPUGovernor(idle_timeout=30)
-
 # ----------------------------------------------------------------------------
 # Config
 # ----------------------------------------------------------------------------
@@ -125,7 +77,6 @@ def est_tokens(text):
     return max(1, (len(text) + 3) // 4)
 
 def _post(path, payload, stream=False):
-    GOVERNOR.wake()
     req = urllib.request.Request(
         BACKEND + path,
         data=json.dumps(payload).encode(),
@@ -153,7 +104,6 @@ def _backend_stream(messages, **kw):
     body.update(kw)
     with _post("/chat/completions", body) as r:
         for raw in r:
-            GOVERNOR.touch()
             line = raw.decode(errors="replace").strip()
             if not line:
                 continue

scripts/power-governor.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 """
-power-governor.py — Dynamic clock and power governor for CMP 50HX GPUs.
+power-governor.py — Resilient Dynamic GPU Power Governor for CMP 50HX.
 
-Directly queries llama-server slot state for 100% accurate active/idle tracking:
-- When generating (is_processing: True): Instantly sets full 1900 MHz boost clocks.
-- When idle (is_processing: False for > 15s): Downclocks CMP cards to 600 MHz (~32W/card).
+Tracks llama-server slot state + GPU utilization with fail-safe active retention:
+- Never downclocks while llama-server is processing a request or busy in CUDA.
+- Holds full 1900 MHz boost clocks for 45s of silence before entering low-power idle.
+- Safe 150W power cap enforced on boot.
 """
 
 import time
@@ -14,7 +15,7 @@ import json
 import sys
 
 CMP_GPUS = "1,2"
-IDLE_TIMEOUT_SEC = 15
+IDLE_TIMEOUT_SEC = 45
 LOW_POWER_CLOCK = 300
 
 def run_cmd(*args):
@@ -23,14 +24,27 @@ def run_cmd(*args):
     except Exception:
         pass
 
+def get_gpu_utilization():
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", f"-i={CMP_GPUS}", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"],
+            universal_newlines=True,
+            timeout=2.0
+        )
+        utils = [int(line.strip()) for line in out.strip().split("\n") if line.strip().isdigit()]
+        return max(utils) if utils else 0
+    except Exception:
+        return 0
+
 def is_server_processing():
     try:
         req = urllib.request.Request("http://127.0.0.1:8080/slots", headers={"User-Agent": "power-governor"})
-        with urllib.request.urlopen(req, timeout=1.0) as r:
+        with urllib.request.urlopen(req, timeout=4.0) as r:
             slots = json.loads(r.read().decode())
             return any(bool(s.get("is_processing")) for s in slots)
     except Exception:
-        return False
+        # If server is busy or timing out, assume it is ACTIVE to prevent downclocking
+        return True
 
 def main():
     print("[power-governor] Initializing GPU persistence and 150W power cap...", flush=True)
@@ -41,10 +55,15 @@ def main():
     is_low_power = False
     last_active = time.time()
 
-    print("[power-governor] Monitoring llama-server processing state...", flush=True)
+    print("[power-governor] Monitoring llama-server state & GPU compute...", flush=True)
     while True:
         try:
             active = is_server_processing()
+            if not active:
+                # Double-check GPU compute utilization before declaring idle
+                if get_gpu_utilization() > 0:
+                    active = True
+
             now = time.time()
 
             if active:
@@ -57,11 +76,11 @@ def main():
                 if not is_low_power and (now - last_active) > IDLE_TIMEOUT_SEC:
                     run_cmd("nvidia-smi", f"-i={CMP_GPUS}", f"-lgc={LOW_POWER_CLOCK},{LOW_POWER_CLOCK}")
                     is_low_power = True
-                    print(f"[{time.strftime('%X')}] Inference Idle -> Low-power state engaged (600 MHz, ~32W/card)", flush=True)
+                    print(f"[{time.strftime('%X')}] Inference Idle (> {IDLE_TIMEOUT_SEC}s) -> Low-power state engaged (300 MHz, ~32W/card)", flush=True)
 
-            time.sleep(0.5 if is_low_power else 1.0)
+            time.sleep(0.5 if is_low_power else 1.5)
         except Exception as e:
-            time.sleep(3.0)
+            time.sleep(2.0)
 
 if __name__ == "__main__":
     main()