From c30f4772f9f7dda9255d04f4b11e77d14fca1601 Mon Sep 17 00:00:00 2001 From: wmantly Date: Tue, 1 Sep 2026 14:40:19 +0000 Subject: [PATCH] Fix power governor active retention during long prompt evaluations and remove redundant proxy governor thread --- scripts/ollama-proxy.py | 50 --------------------------------------- scripts/power-governor.py | 41 +++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 61 deletions(-) diff --git a/scripts/ollama-proxy.py b/scripts/ollama-proxy.py index a00a755..0cb1f44 100644 --- a/scripts/ollama-proxy.py +++ b/scripts/ollama-proxy.py @@ -26,54 +26,6 @@ import threading import re import subprocess -# ---------------------------------------------------------------------------- -# GPU Power & Clock Governor (Automatic idle power reduction for CMP 50HX) -# ---------------------------------------------------------------------------- -class GPUGovernor: - """Manages GPU clocks dynamically: drops CMP cards to low-power state when idle (saves ~90W), - and instantly unconstrains to full boost clocks during inference.""" - def __init__(self, idle_timeout=30): - self.idle_timeout = idle_timeout - self.last_active = time.time() - self.is_low_power = False - self.lock = threading.Lock() - self._init_limits() - t = threading.Thread(target=self._monitor_loop, daemon=True) - t.start() - - def _run_cmd(self, *args): - try: - subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) - except Exception: - pass - - def _init_limits(self): - # Ensure persistence mode and 150W safe power cap - self._run_cmd("nvidia-smi", "-pm", "1") - self._run_cmd("nvidia-smi", "-i", "1,2", "-pl", "150") - - def wake(self): - with self.lock: - self.last_active = time.time() - if self.is_low_power: - self._run_cmd("nvidia-smi", "-rgc") - self.is_low_power = False - - def touch(self): - with self.lock: - self.last_active = time.time() - - def _monitor_loop(self): - while True: - time.sleep(5) - with self.lock: - if not self.is_low_power and (time.time() - self.last_active) > self.idle_timeout: - # Drop CMP cards to low power idle clock - self._run_cmd("nvidia-smi", "-i", "1,2", "-lgc", "600,600") - self.is_low_power = True - -GOVERNOR = GPUGovernor(idle_timeout=30) - # ---------------------------------------------------------------------------- # Config # ---------------------------------------------------------------------------- @@ -125,7 +77,6 @@ def est_tokens(text): return max(1, (len(text) + 3) // 4) def _post(path, payload, stream=False): - GOVERNOR.wake() req = urllib.request.Request( BACKEND + path, data=json.dumps(payload).encode(), @@ -153,7 +104,6 @@ def _backend_stream(messages, **kw): body.update(kw) with _post("/chat/completions", body) as r: for raw in r: - GOVERNOR.touch() line = raw.decode(errors="replace").strip() if not line: continue diff --git a/scripts/power-governor.py b/scripts/power-governor.py index df467fa..5142096 100755 --- a/scripts/power-governor.py +++ b/scripts/power-governor.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 """ -power-governor.py — Dynamic clock and power governor for CMP 50HX GPUs. +power-governor.py — Resilient Dynamic GPU Power Governor for CMP 50HX. -Directly queries llama-server slot state for 100% accurate active/idle tracking: -- When generating (is_processing: True): Instantly sets full 1900 MHz boost clocks. -- When idle (is_processing: False for > 15s): Downclocks CMP cards to 600 MHz (~32W/card). +Tracks llama-server slot state + GPU utilization with fail-safe active retention: +- Never downclocks while llama-server is processing a request or busy in CUDA. +- Holds full 1900 MHz boost clocks for 45s of silence before entering low-power idle. +- Safe 150W power cap enforced on boot. """ import time @@ -14,7 +15,7 @@ import json import sys CMP_GPUS = "1,2" -IDLE_TIMEOUT_SEC = 15 +IDLE_TIMEOUT_SEC = 45 LOW_POWER_CLOCK = 300 def run_cmd(*args): @@ -23,14 +24,27 @@ def run_cmd(*args): except Exception: pass +def get_gpu_utilization(): + try: + out = subprocess.check_output( + ["nvidia-smi", f"-i={CMP_GPUS}", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"], + universal_newlines=True, + timeout=2.0 + ) + utils = [int(line.strip()) for line in out.strip().split("\n") if line.strip().isdigit()] + return max(utils) if utils else 0 + except Exception: + return 0 + def is_server_processing(): try: req = urllib.request.Request("http://127.0.0.1:8080/slots", headers={"User-Agent": "power-governor"}) - with urllib.request.urlopen(req, timeout=1.0) as r: + with urllib.request.urlopen(req, timeout=4.0) as r: slots = json.loads(r.read().decode()) return any(bool(s.get("is_processing")) for s in slots) except Exception: - return False + # If server is busy or timing out, assume it is ACTIVE to prevent downclocking + return True def main(): print("[power-governor] Initializing GPU persistence and 150W power cap...", flush=True) @@ -41,10 +55,15 @@ def main(): is_low_power = False last_active = time.time() - print("[power-governor] Monitoring llama-server processing state...", flush=True) + print("[power-governor] Monitoring llama-server state & GPU compute...", flush=True) while True: try: active = is_server_processing() + if not active: + # Double-check GPU compute utilization before declaring idle + if get_gpu_utilization() > 0: + active = True + now = time.time() if active: @@ -57,11 +76,11 @@ def main(): if not is_low_power and (now - last_active) > IDLE_TIMEOUT_SEC: run_cmd("nvidia-smi", f"-i={CMP_GPUS}", f"-lgc={LOW_POWER_CLOCK},{LOW_POWER_CLOCK}") is_low_power = True - print(f"[{time.strftime('%X')}] Inference Idle -> Low-power state engaged (600 MHz, ~32W/card)", flush=True) + print(f"[{time.strftime('%X')}] Inference Idle (> {IDLE_TIMEOUT_SEC}s) -> Low-power state engaged (300 MHz, ~32W/card)", flush=True) - time.sleep(0.5 if is_low_power else 1.0) + time.sleep(0.5 if is_low_power else 1.5) except Exception as e: - time.sleep(3.0) + time.sleep(2.0) if __name__ == "__main__": main()