feat: initial orchestrator service with FastAPI backend and premium GUI
- FastAPI backend with full Proxmox VE API integration - ISO builder using proxmox-auto-install-assistant - Premium dark-mode SPA frontend with glassmorphism design - VM lifecycle management (create, start, stop, destroy) - Build pipeline tracking with real-time logs - Deployment automation for custom auto-installer ISOs - Production deployment script (setup.sh + systemd) - Comprehensive README with API documentation
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Sovereign Orchestrator App Package
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Sovereign Orchestrator - Configuration Module
|
||||
|
||||
Loads application configuration from environment variables and
|
||||
the ~/.proxmox-credentials file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CREDENTIALS_PATH = Path.home() / ".proxmox-credentials"
|
||||
|
||||
|
||||
def _load_credentials_file() -> dict[str, str]:
|
||||
"""Parse the ~/.proxmox-credentials file into a dict of key=value pairs."""
|
||||
creds: dict[str, str] = {}
|
||||
if not _CREDENTIALS_PATH.exists():
|
||||
logger.warning("Credentials file not found at %s", _CREDENTIALS_PATH)
|
||||
return creds
|
||||
try:
|
||||
for line in _CREDENTIALS_PATH.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" in line:
|
||||
key, _, value = line.partition("=")
|
||||
creds[key.strip()] = value.strip()
|
||||
except OSError as exc:
|
||||
logger.error("Failed to read credentials file: %s", exc)
|
||||
return creds
|
||||
|
||||
|
||||
_creds = _load_credentials_file()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
"""Application settings with env-var overrides."""
|
||||
|
||||
# Proxmox connection
|
||||
proxmox_host: str = os.getenv(
|
||||
"PROXMOX_HOST", _creds.get("PROXMOX_HOST", "https://localhost:8006")
|
||||
)
|
||||
proxmox_token_id: str = os.getenv(
|
||||
"PROXMOX_TOKEN_ID", _creds.get("PROXMOX_TOKEN_ID", "")
|
||||
)
|
||||
proxmox_token_secret: str = os.getenv(
|
||||
"PROXMOX_TOKEN_SECRET", _creds.get("PROXMOX_TOKEN_SECRET", "")
|
||||
)
|
||||
|
||||
# Storage paths
|
||||
iso_storage_path: str = os.getenv(
|
||||
"ISO_STORAGE_PATH", "/var/lib/vz/template/iso"
|
||||
)
|
||||
build_output_path: str = os.getenv(
|
||||
"BUILD_OUTPUT_PATH", "/tmp/sovereign-orchestrator/builds"
|
||||
)
|
||||
|
||||
# Defaults
|
||||
default_node: str = os.getenv("DEFAULT_NODE", "dl380-0")
|
||||
default_vmid: int = int(os.getenv("DEFAULT_VMID", "900"))
|
||||
default_storage: str = os.getenv("DEFAULT_STORAGE", "local")
|
||||
|
||||
# App
|
||||
app_host: str = os.getenv("APP_HOST", "0.0.0.0")
|
||||
app_port: int = int(os.getenv("APP_PORT", "8888"))
|
||||
|
||||
# Reference answers.toml template
|
||||
answers_template_path: str = os.getenv(
|
||||
"ANSWERS_TEMPLATE_PATH",
|
||||
"/home/william/dev/theta42/proxmox-appliance-automation/files/answers.toml",
|
||||
)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,244 @@
|
||||
"""
|
||||
Sovereign Orchestrator - ISO Builder
|
||||
|
||||
Generates answers.toml files and invokes `proxmox-auto-install-assistant`
|
||||
to produce custom auto-installer ISOs. Build state is tracked in-memory
|
||||
and exposed through the API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import toml
|
||||
|
||||
from app.config import settings
|
||||
from app.models import BuildState, BuildStatus, ISOConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# In-memory build registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_builds: dict[str, BuildStatus] = {}
|
||||
|
||||
|
||||
def get_all_builds() -> list[BuildStatus]:
|
||||
"""Return all builds, newest first."""
|
||||
return sorted(
|
||||
_builds.values(), key=lambda b: b.created_at, reverse=True
|
||||
)
|
||||
|
||||
|
||||
def get_build(build_id: str) -> Optional[BuildStatus]:
|
||||
"""Look up a single build by ID."""
|
||||
return _builds.get(build_id)
|
||||
|
||||
|
||||
def get_latest_build() -> Optional[BuildStatus]:
|
||||
"""Return the most recent build, or None."""
|
||||
builds = get_all_builds()
|
||||
return builds[0] if builds else None
|
||||
|
||||
|
||||
def get_active_builds() -> list[BuildStatus]:
|
||||
"""Return builds that are still in progress."""
|
||||
return [
|
||||
b for b in _builds.values()
|
||||
if b.state in (BuildState.PENDING, BuildState.BUILDING, BuildState.UPLOADING)
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# answers.toml generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_answers_toml(config: ISOConfig) -> str:
|
||||
"""Render an ``answers.toml`` string from the given configuration.
|
||||
|
||||
The output format matches the Proxmox auto-installer specification
|
||||
with [global], [network], and [disk-setup] sections.
|
||||
"""
|
||||
doc: dict = {
|
||||
"global": {
|
||||
"keyboard": config.keyboard,
|
||||
"country": config.country,
|
||||
"timezone": config.timezone,
|
||||
"mailto": config.mailto,
|
||||
"root-password": config.root_password,
|
||||
"fqdn": config.fqdn,
|
||||
"reboot-on-error": config.reboot_on_error,
|
||||
},
|
||||
"network": {
|
||||
"source": config.network_source.value,
|
||||
},
|
||||
"disk-setup": {
|
||||
"filesystem": config.filesystem.value,
|
||||
"disk-list": config.disk_list,
|
||||
},
|
||||
}
|
||||
|
||||
if config.root_ssh_keys:
|
||||
doc["global"]["root-ssh-keys"] = config.root_ssh_keys
|
||||
|
||||
return toml.dumps(doc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base ISO discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BASE_ISO_SEARCH_PATHS = [
|
||||
Path(settings.iso_storage_path),
|
||||
Path("/var/lib/vz/template/iso"),
|
||||
Path("/usr/share/proxmox-installer"),
|
||||
Path.home() / "isos",
|
||||
]
|
||||
|
||||
|
||||
def find_base_iso() -> Optional[Path]:
|
||||
"""Locate the first Proxmox VE ISO in well-known directories.
|
||||
|
||||
Searches for files matching ``proxmox-ve_*.iso``.
|
||||
"""
|
||||
for search_dir in _BASE_ISO_SEARCH_PATHS:
|
||||
if not search_dir.is_dir():
|
||||
continue
|
||||
candidates = sorted(
|
||||
search_dir.glob("proxmox-ve_*.iso"), reverse=True
|
||||
)
|
||||
if candidates:
|
||||
logger.info("Found base ISO: %s", candidates[0])
|
||||
return candidates[0]
|
||||
logger.warning("No base Proxmox VE ISO found in search paths")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ISO build pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ensure_build_dir() -> Path:
|
||||
"""Create and return the build output directory."""
|
||||
build_dir = Path(settings.build_output_path)
|
||||
build_dir.mkdir(parents=True, exist_ok=True)
|
||||
return build_dir
|
||||
|
||||
|
||||
async def build_iso(config: ISOConfig) -> BuildStatus:
|
||||
"""Kick off an asynchronous ISO build and return the BuildStatus.
|
||||
|
||||
The actual subprocess work runs in the background so the API can
|
||||
return immediately.
|
||||
"""
|
||||
build = BuildStatus(iso_config=config)
|
||||
_builds[build.id] = build
|
||||
build.log(f"Build {build.id} created")
|
||||
|
||||
# Launch the background task
|
||||
asyncio.create_task(_run_build(build))
|
||||
|
||||
return build
|
||||
|
||||
|
||||
async def _run_build(build: BuildStatus) -> None:
|
||||
"""Execute the full ISO build pipeline."""
|
||||
try:
|
||||
build.state = BuildState.BUILDING
|
||||
build.log("Build started")
|
||||
|
||||
build_dir = _ensure_build_dir()
|
||||
work_dir = build_dir / build.id
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. Write answers.toml
|
||||
answers_path = work_dir / "answers.toml"
|
||||
answers_content = generate_answers_toml(build.iso_config)
|
||||
answers_path.write_text(answers_content)
|
||||
build.log(f"answers.toml written to {answers_path}")
|
||||
logger.debug("answers.toml content:\n%s", answers_content)
|
||||
|
||||
# 2. Locate base ISO
|
||||
base_iso = find_base_iso()
|
||||
if base_iso is None:
|
||||
raise FileNotFoundError(
|
||||
"No base Proxmox VE ISO found. Place a proxmox-ve_*.iso in "
|
||||
f"{settings.iso_storage_path}"
|
||||
)
|
||||
build.log(f"Using base ISO: {base_iso}")
|
||||
|
||||
# 3. Determine output filename
|
||||
fqdn_slug = build.iso_config.fqdn.replace(".", "-")
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
output_filename = f"pve-autoinstall-{fqdn_slug}-{timestamp}.iso"
|
||||
output_path = work_dir / output_filename
|
||||
|
||||
# 4. Copy the base ISO to the output path first (prepare-iso works
|
||||
# on a copy so we don't mutate the original)
|
||||
build.log(f"Copying base ISO to {output_path}")
|
||||
await asyncio.to_thread(shutil.copy2, str(base_iso), str(output_path))
|
||||
build.log("Base ISO copied")
|
||||
|
||||
# 5. Run proxmox-auto-install-assistant prepare-iso
|
||||
cmd = [
|
||||
"proxmox-auto-install-assistant",
|
||||
"prepare-iso",
|
||||
str(output_path),
|
||||
"--fetch-from", "iso",
|
||||
"--answer-file", str(answers_path),
|
||||
]
|
||||
build.log(f"Running: {' '.join(cmd)}")
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
cwd=str(work_dir),
|
||||
)
|
||||
|
||||
# Stream output into build logs
|
||||
assert proc.stdout is not None
|
||||
while True:
|
||||
line = await proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
decoded = line.decode("utf-8", errors="replace").rstrip()
|
||||
build.log(decoded)
|
||||
logger.debug("[build %s] %s", build.id, decoded)
|
||||
|
||||
return_code = await proc.wait()
|
||||
|
||||
if return_code != 0:
|
||||
raise RuntimeError(
|
||||
f"proxmox-auto-install-assistant exited with code {return_code}"
|
||||
)
|
||||
|
||||
build.iso_filename = output_filename
|
||||
build.state = BuildState.COMPLETED
|
||||
build.log(f"Build completed: {output_filename}")
|
||||
logger.info("Build %s completed: %s", build.id, output_path)
|
||||
|
||||
except Exception as exc:
|
||||
build.state = BuildState.FAILED
|
||||
build.error = str(exc)
|
||||
build.log(f"Build failed: {exc}")
|
||||
logger.exception("Build %s failed", build.id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utility: get the output ISO path for a build
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_build_iso_path(build_id: str) -> Optional[Path]:
|
||||
"""Return the absolute path to the output ISO for a completed build."""
|
||||
build = _builds.get(build_id)
|
||||
if not build or not build.iso_filename:
|
||||
return None
|
||||
return Path(settings.build_output_path) / build.id / build.iso_filename
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
"""
|
||||
Sovereign Orchestrator - FastAPI Application
|
||||
|
||||
Main entrypoint that wires up all API routes, middleware, and static file
|
||||
serving. Run with:
|
||||
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 8888
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from app.config import settings
|
||||
from app.iso_builder import (
|
||||
build_iso,
|
||||
get_all_builds,
|
||||
get_build,
|
||||
get_build_iso_path,
|
||||
get_active_builds,
|
||||
get_latest_build,
|
||||
)
|
||||
from app.models import (
|
||||
BuildState,
|
||||
BuildStatus,
|
||||
DeployConfig,
|
||||
DeployResult,
|
||||
ISOConfig,
|
||||
ISOInfo,
|
||||
NodeInfo,
|
||||
SystemStatus,
|
||||
VMStatus,
|
||||
)
|
||||
from app.proxmox_client import ProxmoxError, proxmox
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-8s %(name)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App setup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app = FastAPI(
|
||||
title="Sovereign Orchestrator",
|
||||
description=(
|
||||
"API for building Proxmox auto-install ISOs and deploying "
|
||||
"VMs on a Proxmox VE cluster."
|
||||
),
|
||||
version="0.1.0",
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files – mount the frontend at / (must be last, after API routes)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_static_dir = Path(__file__).resolve().parent.parent / "static"
|
||||
_static_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Health / status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/status", response_model=SystemStatus, tags=["Status"])
|
||||
async def get_system_status() -> SystemStatus:
|
||||
"""Return an aggregate system status dashboard.
|
||||
|
||||
Includes Proxmox node info, test VM state, available ISOs, and the
|
||||
active build queue. Proxmox errors are captured gracefully so the
|
||||
endpoint always returns *something*.
|
||||
"""
|
||||
status = SystemStatus()
|
||||
|
||||
try:
|
||||
# Nodes
|
||||
raw_nodes = await proxmox.get_nodes()
|
||||
status.nodes = [
|
||||
NodeInfo(
|
||||
node=n.get("node", "unknown"),
|
||||
status=n.get("status", "unknown"),
|
||||
cpu=n.get("cpu"),
|
||||
maxcpu=n.get("maxcpu"),
|
||||
mem=n.get("mem"),
|
||||
maxmem=n.get("maxmem"),
|
||||
uptime=n.get("uptime"),
|
||||
)
|
||||
for n in raw_nodes
|
||||
]
|
||||
|
||||
# Test VM
|
||||
try:
|
||||
vm_data = await proxmox.get_vm_status()
|
||||
status.test_vm = VMStatus(
|
||||
vmid=settings.default_vmid,
|
||||
name=vm_data.get("name"),
|
||||
status=vm_data.get("status", "unknown"),
|
||||
node=settings.default_node,
|
||||
cpus=vm_data.get("cpus"),
|
||||
maxmem=vm_data.get("maxmem"),
|
||||
maxdisk=vm_data.get("maxdisk"),
|
||||
uptime=vm_data.get("uptime"),
|
||||
pid=vm_data.get("pid"),
|
||||
)
|
||||
except ProxmoxError:
|
||||
logger.debug("Test VM %d not found (may not exist yet)", settings.default_vmid)
|
||||
|
||||
# ISOs
|
||||
try:
|
||||
raw_isos = await proxmox.get_isos()
|
||||
status.available_isos = [
|
||||
ISOInfo(
|
||||
volid=iso.get("volid", ""),
|
||||
filename=iso.get("volid", "").split("/")[-1],
|
||||
size=iso.get("size"),
|
||||
)
|
||||
for iso in raw_isos
|
||||
]
|
||||
except ProxmoxError:
|
||||
logger.debug("Could not list ISOs")
|
||||
|
||||
status.proxmox_connected = True
|
||||
|
||||
except ProxmoxError as exc:
|
||||
status.error = str(exc)
|
||||
logger.warning("Proxmox status check failed: %s", exc)
|
||||
except Exception as exc:
|
||||
status.error = f"Unexpected error: {exc}"
|
||||
logger.exception("Status check failed")
|
||||
|
||||
# Build queue (always available even if Proxmox is down)
|
||||
status.active_builds = get_active_builds()
|
||||
|
||||
return status
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ISO generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/api/generate-iso", response_model=BuildStatus, tags=["ISO"])
|
||||
async def generate_iso(config: ISOConfig) -> BuildStatus:
|
||||
"""Generate a custom Proxmox auto-install ISO.
|
||||
|
||||
Accepts the installation configuration, writes an ``answers.toml``,
|
||||
and asynchronously runs ``proxmox-auto-install-assistant prepare-iso``.
|
||||
Returns the build status immediately; poll ``/api/builds/{id}`` for
|
||||
progress.
|
||||
"""
|
||||
logger.info("ISO generation requested for %s", config.fqdn)
|
||||
build = await build_iso(config)
|
||||
return build
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Builds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/builds", response_model=list[BuildStatus], tags=["Builds"])
|
||||
async def list_builds() -> list[BuildStatus]:
|
||||
"""List all ISO builds, newest first."""
|
||||
return get_all_builds()
|
||||
|
||||
|
||||
@app.get("/api/builds/{build_id}", response_model=BuildStatus, tags=["Builds"])
|
||||
async def get_build_detail(build_id: str) -> BuildStatus:
|
||||
"""Get detailed status and logs for a specific build."""
|
||||
build = get_build(build_id)
|
||||
if build is None:
|
||||
raise HTTPException(status_code=404, detail=f"Build {build_id} not found")
|
||||
return build
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deploy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/api/deploy", response_model=DeployResult, tags=["Deploy"])
|
||||
async def deploy_vm(config: DeployConfig) -> DeployResult:
|
||||
"""Deploy a VM using the custom auto-install ISO.
|
||||
|
||||
This will:
|
||||
1. Resolve which ISO to use (from *build_id* or the latest build)
|
||||
2. Upload the ISO to Proxmox storage if needed
|
||||
3. Stop and destroy the existing VM (if any)
|
||||
4. Create a new VM with the ISO mounted
|
||||
5. Start the VM
|
||||
|
||||
The entire flow runs synchronously so the caller gets a final status.
|
||||
"""
|
||||
node = config.node
|
||||
vmid = config.vmid
|
||||
|
||||
# --- Resolve the ISO to deploy ---
|
||||
build_id = config.build_id
|
||||
if build_id is None:
|
||||
latest = get_latest_build()
|
||||
if latest is None or latest.state != BuildState.COMPLETED:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No completed build available. Run /api/generate-iso first.",
|
||||
)
|
||||
build_id = latest.id
|
||||
|
||||
build = get_build(build_id)
|
||||
if build is None or build.state != BuildState.COMPLETED:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Build {build_id} is not in a completed state.",
|
||||
)
|
||||
|
||||
iso_path = get_build_iso_path(build_id)
|
||||
if iso_path is None or not iso_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"ISO file for build {build_id} not found on disk.",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Deploying VM %d on %s from build %s (%s)",
|
||||
vmid, node, build_id, iso_path.name,
|
||||
)
|
||||
|
||||
try:
|
||||
# --- Upload ISO to Proxmox ---
|
||||
iso_volid = f"{config.iso_storage}:iso/{iso_path.name}"
|
||||
try:
|
||||
existing_isos = await proxmox.get_isos(node, config.iso_storage)
|
||||
already_uploaded = any(
|
||||
i.get("volid") == iso_volid for i in existing_isos
|
||||
)
|
||||
except ProxmoxError:
|
||||
already_uploaded = False
|
||||
|
||||
if not already_uploaded:
|
||||
logger.info("Uploading ISO %s to Proxmox storage", iso_path.name)
|
||||
upload_upid = await proxmox.upload_iso(
|
||||
str(iso_path), node, config.iso_storage
|
||||
)
|
||||
await proxmox.wait_for_task(node, upload_upid, max_wait=300)
|
||||
logger.info("ISO upload completed")
|
||||
|
||||
# --- Tear down existing VM ---
|
||||
try:
|
||||
vm_status = await proxmox.get_vm_status(node, vmid)
|
||||
current_state = vm_status.get("status", "")
|
||||
|
||||
if current_state == "running":
|
||||
logger.info("Stopping existing VM %d", vmid)
|
||||
stop_upid = await proxmox.stop_vm(node, vmid)
|
||||
await proxmox.wait_for_task(node, stop_upid, max_wait=60)
|
||||
# Brief pause for lock release
|
||||
await asyncio.sleep(3)
|
||||
|
||||
logger.info("Destroying existing VM %d", vmid)
|
||||
destroy_upid = await proxmox.destroy_vm(node, vmid)
|
||||
await proxmox.wait_for_task(node, destroy_upid, max_wait=60)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
except ProxmoxError as exc:
|
||||
if exc.status_code == 500 or exc.status_code == 404:
|
||||
logger.info("VM %d does not exist, skipping teardown", vmid)
|
||||
else:
|
||||
raise
|
||||
|
||||
# --- Create new VM ---
|
||||
vm_config: dict[str, Any] = {
|
||||
"vmid": vmid,
|
||||
"name": build.iso_config.fqdn.split(".")[0] if build.iso_config else f"vm-{vmid}",
|
||||
"cores": config.cores,
|
||||
"memory": config.memory,
|
||||
"ostype": "l26",
|
||||
"scsihw": "virtio-scsi-single",
|
||||
"scsi0": f"{config.storage}:{config.disk_size.rstrip('G')},iothread=1",
|
||||
"ide2": f"{iso_volid},media=cdrom",
|
||||
"boot": "order=ide2;scsi0",
|
||||
"net0": "virtio,bridge=vmbr0",
|
||||
"serial0": "socket",
|
||||
"vga": "serial0",
|
||||
}
|
||||
|
||||
create_upid = await proxmox.create_vm(node, vmid, vm_config)
|
||||
if isinstance(create_upid, str) and create_upid.startswith("UPID"):
|
||||
await proxmox.wait_for_task(node, create_upid, max_wait=30)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# --- Start the VM ---
|
||||
start_upid = await proxmox.start_vm(node, vmid)
|
||||
|
||||
return DeployResult(
|
||||
vmid=vmid,
|
||||
node=node,
|
||||
status="started",
|
||||
message=f"VM {vmid} created and started with ISO {iso_path.name}",
|
||||
task_id=start_upid if isinstance(start_upid, str) else None,
|
||||
)
|
||||
|
||||
except ProxmoxError as exc:
|
||||
logger.error("Deploy failed: %s", exc)
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
except Exception as exc:
|
||||
logger.exception("Unexpected deploy error")
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VM management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/vm/{vmid}/status", response_model=VMStatus, tags=["VM"])
|
||||
async def vm_status(vmid: int, node: str | None = None) -> VMStatus:
|
||||
"""Get the current status of a VM."""
|
||||
node = node or settings.default_node
|
||||
try:
|
||||
data = await proxmox.get_vm_status(node, vmid)
|
||||
return VMStatus(
|
||||
vmid=vmid,
|
||||
name=data.get("name"),
|
||||
status=data.get("status", "unknown"),
|
||||
node=node,
|
||||
cpus=data.get("cpus"),
|
||||
maxmem=data.get("maxmem"),
|
||||
maxdisk=data.get("maxdisk"),
|
||||
uptime=data.get("uptime"),
|
||||
pid=data.get("pid"),
|
||||
)
|
||||
except ProxmoxError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
|
||||
|
||||
@app.post("/api/vm/{vmid}/start", tags=["VM"])
|
||||
async def vm_start(vmid: int, node: str | None = None) -> dict[str, str]:
|
||||
"""Start a VM."""
|
||||
node = node or settings.default_node
|
||||
try:
|
||||
task_id = await proxmox.start_vm(node, vmid)
|
||||
return {"status": "starting", "vmid": str(vmid), "task_id": str(task_id)}
|
||||
except ProxmoxError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
|
||||
|
||||
@app.post("/api/vm/{vmid}/stop", tags=["VM"])
|
||||
async def vm_stop(vmid: int, node: str | None = None) -> dict[str, str]:
|
||||
"""Stop a VM."""
|
||||
node = node or settings.default_node
|
||||
try:
|
||||
task_id = await proxmox.stop_vm(node, vmid)
|
||||
return {"status": "stopping", "vmid": str(vmid), "task_id": str(task_id)}
|
||||
except ProxmoxError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ISOs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/isos", response_model=list[ISOInfo], tags=["ISO"])
|
||||
async def list_isos(
|
||||
node: str | None = None, storage: str | None = None
|
||||
) -> list[ISOInfo]:
|
||||
"""List available ISO images on Proxmox storage."""
|
||||
try:
|
||||
raw = await proxmox.get_isos(node, storage)
|
||||
return [
|
||||
ISOInfo(
|
||||
volid=iso.get("volid", ""),
|
||||
filename=iso.get("volid", "").split("/")[-1],
|
||||
size=iso.get("size"),
|
||||
)
|
||||
for iso in raw
|
||||
]
|
||||
except ProxmoxError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files — mounted last so API routes take precedence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app.mount("/", StaticFiles(directory=str(_static_dir), html=True), name="static")
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
Sovereign Orchestrator - Pydantic Models
|
||||
|
||||
Data models for API request/response payloads and internal state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BuildState(str, Enum):
|
||||
"""Lifecycle states for an ISO build job."""
|
||||
PENDING = "pending"
|
||||
BUILDING = "building"
|
||||
UPLOADING = "uploading"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class NetworkSource(str, Enum):
|
||||
"""Proxmox installer network source options."""
|
||||
FROM_DHCP = "from-dhcp"
|
||||
FROM_ANSWER = "from-answer"
|
||||
|
||||
|
||||
class Filesystem(str, Enum):
|
||||
"""Supported root filesystems."""
|
||||
EXT4 = "ext4"
|
||||
XFS = "xfs"
|
||||
ZFS = "zfs"
|
||||
BTRFS = "btrfs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ISOConfig(BaseModel):
|
||||
"""Configuration payload for generating a custom auto-install ISO."""
|
||||
|
||||
fqdn: str = Field(
|
||||
..., description="Fully qualified domain name for the new host",
|
||||
examples=["pve-test.internal.718it.biz"],
|
||||
)
|
||||
keyboard: str = Field(default="en-us", description="Keyboard layout")
|
||||
country: str = Field(default="us", description="Country code")
|
||||
timezone: str = Field(
|
||||
default="America/New_York", description="Timezone string"
|
||||
)
|
||||
mailto: str = Field(
|
||||
default="wmantly@gmail.com", description="Admin email address"
|
||||
)
|
||||
root_password: str = Field(
|
||||
..., description="Root password for the installed system"
|
||||
)
|
||||
root_ssh_keys: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="SSH public keys to authorize for root",
|
||||
)
|
||||
network_source: NetworkSource = Field(
|
||||
default=NetworkSource.FROM_DHCP,
|
||||
description="Network configuration source",
|
||||
)
|
||||
filesystem: Filesystem = Field(
|
||||
default=Filesystem.EXT4, description="Root filesystem type"
|
||||
)
|
||||
disk_list: list[str] = Field(
|
||||
default_factory=lambda: ["sda"],
|
||||
description="Disks to use for installation",
|
||||
)
|
||||
reboot_on_error: bool = Field(
|
||||
default=False, description="Reboot automatically on install error"
|
||||
)
|
||||
|
||||
|
||||
class DeployConfig(BaseModel):
|
||||
"""Configuration for deploying a VM with the custom ISO."""
|
||||
|
||||
vmid: int = Field(default=900, description="VM ID to create")
|
||||
node: str = Field(default="dl380-0", description="Proxmox node name")
|
||||
cores: int = Field(default=4, ge=1, le=128, description="CPU cores")
|
||||
memory: int = Field(
|
||||
default=8192, ge=512, description="Memory in MiB"
|
||||
)
|
||||
disk_size: str = Field(
|
||||
default="64G", description="Root disk size (e.g. 64G)"
|
||||
)
|
||||
build_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Build ID whose ISO to use. Defaults to latest.",
|
||||
)
|
||||
storage: str = Field(
|
||||
default="local-lvm", description="Proxmox storage for the VM disk"
|
||||
)
|
||||
iso_storage: str = Field(
|
||||
default="local", description="Proxmox storage containing ISOs"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal / response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BuildStatus(BaseModel):
|
||||
"""Tracks the state of an ISO build job."""
|
||||
|
||||
id: str = Field(default_factory=lambda: uuid.uuid4().hex[:12])
|
||||
state: BuildState = BuildState.PENDING
|
||||
iso_config: Optional[ISOConfig] = None
|
||||
iso_filename: Optional[str] = None
|
||||
logs: list[str] = Field(default_factory=list)
|
||||
created_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
error: Optional[str] = None
|
||||
|
||||
def log(self, message: str) -> None:
|
||||
"""Append a timestamped log line."""
|
||||
ts = datetime.now(timezone.utc).isoformat()
|
||||
self.logs.append(f"[{ts}] {message}")
|
||||
self.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class VMStatus(BaseModel):
|
||||
"""Proxmox VM status snapshot."""
|
||||
|
||||
vmid: int
|
||||
name: Optional[str] = None
|
||||
status: str
|
||||
node: str
|
||||
cpus: Optional[int] = None
|
||||
maxmem: Optional[int] = None
|
||||
maxdisk: Optional[int] = None
|
||||
uptime: Optional[int] = None
|
||||
pid: Optional[int] = None
|
||||
|
||||
|
||||
class NodeInfo(BaseModel):
|
||||
"""Proxmox node summary."""
|
||||
|
||||
node: str
|
||||
status: str
|
||||
cpu: Optional[float] = None
|
||||
maxcpu: Optional[int] = None
|
||||
mem: Optional[int] = None
|
||||
maxmem: Optional[int] = None
|
||||
uptime: Optional[int] = None
|
||||
|
||||
|
||||
class ISOInfo(BaseModel):
|
||||
"""ISO file metadata."""
|
||||
|
||||
volid: str
|
||||
filename: str
|
||||
size: Optional[int] = None
|
||||
|
||||
|
||||
class SystemStatus(BaseModel):
|
||||
"""Aggregate system status returned by /api/status."""
|
||||
|
||||
nodes: list[NodeInfo] = Field(default_factory=list)
|
||||
test_vm: Optional[VMStatus] = None
|
||||
available_isos: list[ISOInfo] = Field(default_factory=list)
|
||||
active_builds: list[BuildStatus] = Field(default_factory=list)
|
||||
proxmox_connected: bool = False
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class DeployResult(BaseModel):
|
||||
"""Response from a deploy operation."""
|
||||
|
||||
vmid: int
|
||||
node: str
|
||||
status: str
|
||||
message: str
|
||||
task_id: Optional[str] = None
|
||||
@@ -0,0 +1,287 @@
|
||||
"""
|
||||
Sovereign Orchestrator - Proxmox API Client
|
||||
|
||||
Async HTTP client for the Proxmox VE REST API using token-based
|
||||
authentication. All methods return parsed JSON dicts or raise on error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ProxmoxError(Exception):
|
||||
"""Raised when a Proxmox API call fails."""
|
||||
|
||||
def __init__(self, message: str, status_code: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class ProxmoxClient:
|
||||
"""Async Proxmox VE API client with token authentication."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str | None = None,
|
||||
token_id: str | None = None,
|
||||
token_secret: str | None = None,
|
||||
verify_ssl: bool = False,
|
||||
timeout: float = 30.0,
|
||||
) -> None:
|
||||
self.host = (host or settings.proxmox_host).rstrip("/")
|
||||
self.token_id = token_id or settings.proxmox_token_id
|
||||
self.token_secret = token_secret or settings.proxmox_token_secret
|
||||
self.verify_ssl = verify_ssl
|
||||
self.timeout = timeout
|
||||
|
||||
if not self.token_id or not self.token_secret:
|
||||
logger.warning(
|
||||
"Proxmox token credentials are not configured. "
|
||||
"API calls will fail."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _auth_header(self) -> dict[str, str]:
|
||||
"""Build the PVEAPIToken authorization header."""
|
||||
return {
|
||||
"Authorization": f"PVEAPIToken={self.token_id}={self.token_secret}"
|
||||
}
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
"""Build the full API URL for *path* (should start with /)."""
|
||||
return f"{self.host}/api2/json{path}"
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
params: dict[str, Any] | None = None,
|
||||
data: dict[str, Any] | None = None,
|
||||
json_body: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Execute an API request and return the ``data`` envelope."""
|
||||
url = self._url(path)
|
||||
headers = self._auth_header()
|
||||
|
||||
logger.debug("%s %s params=%s", method, url, params)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
verify=self.verify_ssl, timeout=self.timeout
|
||||
) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
data=data,
|
||||
json=json_body,
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
body = response.text[:500]
|
||||
logger.error(
|
||||
"Proxmox API error: %s %s -> %d: %s",
|
||||
method, path, response.status_code, body,
|
||||
)
|
||||
raise ProxmoxError(
|
||||
f"Proxmox API {method} {path} returned {response.status_code}: {body}",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
|
||||
payload = response.json()
|
||||
return payload.get("data", payload)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Node operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_nodes(self) -> list[dict[str, Any]]:
|
||||
"""List all cluster nodes."""
|
||||
return await self._request("GET", "/nodes")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# VM operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_vms(self, node: str | None = None) -> list[dict[str, Any]]:
|
||||
"""List VMs on *node*."""
|
||||
node = node or settings.default_node
|
||||
return await self._request("GET", f"/nodes/{node}/qemu")
|
||||
|
||||
async def get_vm_status(
|
||||
self, node: str | None = None, vmid: int | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Get current status of a single VM."""
|
||||
node = node or settings.default_node
|
||||
vmid = vmid if vmid is not None else settings.default_vmid
|
||||
return await self._request(
|
||||
"GET", f"/nodes/{node}/qemu/{vmid}/status/current"
|
||||
)
|
||||
|
||||
async def start_vm(
|
||||
self, node: str | None = None, vmid: int | None = None
|
||||
) -> str:
|
||||
"""Start a VM. Returns the Proxmox UPID task string."""
|
||||
node = node or settings.default_node
|
||||
vmid = vmid if vmid is not None else settings.default_vmid
|
||||
logger.info("Starting VM %d on %s", vmid, node)
|
||||
return await self._request(
|
||||
"POST", f"/nodes/{node}/qemu/{vmid}/status/start"
|
||||
)
|
||||
|
||||
async def stop_vm(
|
||||
self, node: str | None = None, vmid: int | None = None
|
||||
) -> str:
|
||||
"""Stop (hard) a VM. Returns the Proxmox UPID task string."""
|
||||
node = node or settings.default_node
|
||||
vmid = vmid if vmid is not None else settings.default_vmid
|
||||
logger.info("Stopping VM %d on %s", vmid, node)
|
||||
return await self._request(
|
||||
"POST", f"/nodes/{node}/qemu/{vmid}/status/stop"
|
||||
)
|
||||
|
||||
async def destroy_vm(
|
||||
self,
|
||||
node: str | None = None,
|
||||
vmid: int | None = None,
|
||||
purge: bool = True,
|
||||
destroy_unreferenced_disks: bool = True,
|
||||
) -> str:
|
||||
"""Delete a VM and optionally purge its disks."""
|
||||
node = node or settings.default_node
|
||||
vmid = vmid if vmid is not None else settings.default_vmid
|
||||
logger.info("Destroying VM %d on %s (purge=%s)", vmid, node, purge)
|
||||
params: dict[str, Any] = {}
|
||||
if purge:
|
||||
params["purge"] = 1
|
||||
if destroy_unreferenced_disks:
|
||||
params["destroy-unreferenced-disks"] = 1
|
||||
return await self._request(
|
||||
"DELETE", f"/nodes/{node}/qemu/{vmid}", params=params
|
||||
)
|
||||
|
||||
async def create_vm(
|
||||
self,
|
||||
node: str | None = None,
|
||||
vmid: int | None = None,
|
||||
config: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""Create a new VM with the given configuration dict."""
|
||||
node = node or settings.default_node
|
||||
vmid = vmid if vmid is not None else settings.default_vmid
|
||||
config = config or {}
|
||||
config.setdefault("vmid", vmid)
|
||||
logger.info("Creating VM %d on %s with config: %s", vmid, node, config)
|
||||
return await self._request(
|
||||
"POST", f"/nodes/{node}/qemu", data=config
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ISO / storage operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_isos(
|
||||
self,
|
||||
node: str | None = None,
|
||||
storage: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List ISO images available on *storage*."""
|
||||
node = node or settings.default_node
|
||||
storage = storage or settings.default_storage
|
||||
result = await self._request(
|
||||
"GET",
|
||||
f"/nodes/{node}/storage/{storage}/content",
|
||||
params={"content": "iso"},
|
||||
)
|
||||
# The API returns all content types; filter to ISOs just in case
|
||||
if isinstance(result, list):
|
||||
return [
|
||||
item for item in result
|
||||
if item.get("content") == "iso"
|
||||
or item.get("volid", "").endswith(".iso")
|
||||
]
|
||||
return result
|
||||
|
||||
async def upload_iso(
|
||||
self,
|
||||
filepath: str,
|
||||
node: str | None = None,
|
||||
storage: str | None = None,
|
||||
) -> str:
|
||||
"""Upload a local ISO file to Proxmox storage.
|
||||
|
||||
Uses the Proxmox upload endpoint which expects multipart form data.
|
||||
"""
|
||||
node = node or settings.default_node
|
||||
storage = storage or settings.default_storage
|
||||
url = self._url(f"/nodes/{node}/storage/{storage}/upload")
|
||||
headers = self._auth_header()
|
||||
|
||||
logger.info("Uploading ISO %s to %s:%s", filepath, node, storage)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
verify=self.verify_ssl, timeout=300.0
|
||||
) as client:
|
||||
with open(filepath, "rb") as fh:
|
||||
files = {"filename": (filepath.split("/")[-1], fh, "application/octet-stream")}
|
||||
data = {"content": "iso"}
|
||||
response = await client.post(
|
||||
url, headers=headers, data=data, files=files
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
body = response.text[:500]
|
||||
raise ProxmoxError(
|
||||
f"ISO upload failed ({response.status_code}): {body}",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
|
||||
payload = response.json()
|
||||
task_id = payload.get("data", "")
|
||||
logger.info("Upload task started: %s", task_id)
|
||||
return task_id
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Task helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def wait_for_task(
|
||||
self,
|
||||
node: str,
|
||||
upid: str,
|
||||
*,
|
||||
poll_interval: float = 2.0,
|
||||
max_wait: float = 120.0,
|
||||
) -> dict[str, Any]:
|
||||
"""Poll a Proxmox task until it completes or times out."""
|
||||
import asyncio
|
||||
|
||||
elapsed = 0.0
|
||||
while elapsed < max_wait:
|
||||
status = await self._request(
|
||||
"GET", f"/nodes/{node}/tasks/{upid}/status"
|
||||
)
|
||||
if status.get("status") == "stopped":
|
||||
return status
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
raise ProxmoxError(
|
||||
f"Task {upid} did not complete within {max_wait}s"
|
||||
)
|
||||
|
||||
|
||||
# Module-level singleton for convenience
|
||||
proxmox = ProxmoxClient()
|
||||
Reference in New Issue
Block a user