wmantly
70c71161f3
- FastAPI backend with full Proxmox VE API integration - ISO builder using proxmox-auto-install-assistant - Premium dark-mode SPA frontend with glassmorphism design - VM lifecycle management (create, start, stop, destroy) - Build pipeline tracking with real-time logs - Deployment automation for custom auto-installer ISOs - Production deployment script (setup.sh + systemd) - Comprehensive README with API documentation
404 lines
14 KiB
Python
404 lines
14 KiB
Python
"""
|
||
Sovereign Orchestrator - FastAPI Application
|
||
|
||
Main entrypoint that wires up all API routes, middleware, and static file
|
||
serving. Run with:
|
||
|
||
uvicorn app.main:app --host 0.0.0.0 --port 8888
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from fastapi import FastAPI, HTTPException
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import JSONResponse
|
||
from fastapi.staticfiles import StaticFiles
|
||
|
||
from app.config import settings
|
||
from app.iso_builder import (
|
||
build_iso,
|
||
get_all_builds,
|
||
get_build,
|
||
get_build_iso_path,
|
||
get_active_builds,
|
||
get_latest_build,
|
||
)
|
||
from app.models import (
|
||
BuildState,
|
||
BuildStatus,
|
||
DeployConfig,
|
||
DeployResult,
|
||
ISOConfig,
|
||
ISOInfo,
|
||
NodeInfo,
|
||
SystemStatus,
|
||
VMStatus,
|
||
)
|
||
from app.proxmox_client import ProxmoxError, proxmox
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Logging
|
||
# ---------------------------------------------------------------------------
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s %(levelname)-8s %(name)s %(message)s",
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# App setup
|
||
# ---------------------------------------------------------------------------
|
||
|
||
app = FastAPI(
|
||
title="Sovereign Orchestrator",
|
||
description=(
|
||
"API for building Proxmox auto-install ISOs and deploying "
|
||
"VMs on a Proxmox VE cluster."
|
||
),
|
||
version="0.1.0",
|
||
)
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=["*"],
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Static files – mount the frontend at / (must be last, after API routes)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_static_dir = Path(__file__).resolve().parent.parent / "static"
|
||
_static_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Health / status
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.get("/api/status", response_model=SystemStatus, tags=["Status"])
|
||
async def get_system_status() -> SystemStatus:
|
||
"""Return an aggregate system status dashboard.
|
||
|
||
Includes Proxmox node info, test VM state, available ISOs, and the
|
||
active build queue. Proxmox errors are captured gracefully so the
|
||
endpoint always returns *something*.
|
||
"""
|
||
status = SystemStatus()
|
||
|
||
try:
|
||
# Nodes
|
||
raw_nodes = await proxmox.get_nodes()
|
||
status.nodes = [
|
||
NodeInfo(
|
||
node=n.get("node", "unknown"),
|
||
status=n.get("status", "unknown"),
|
||
cpu=n.get("cpu"),
|
||
maxcpu=n.get("maxcpu"),
|
||
mem=n.get("mem"),
|
||
maxmem=n.get("maxmem"),
|
||
uptime=n.get("uptime"),
|
||
)
|
||
for n in raw_nodes
|
||
]
|
||
|
||
# Test VM
|
||
try:
|
||
vm_data = await proxmox.get_vm_status()
|
||
status.test_vm = VMStatus(
|
||
vmid=settings.default_vmid,
|
||
name=vm_data.get("name"),
|
||
status=vm_data.get("status", "unknown"),
|
||
node=settings.default_node,
|
||
cpus=vm_data.get("cpus"),
|
||
maxmem=vm_data.get("maxmem"),
|
||
maxdisk=vm_data.get("maxdisk"),
|
||
uptime=vm_data.get("uptime"),
|
||
pid=vm_data.get("pid"),
|
||
)
|
||
except ProxmoxError:
|
||
logger.debug("Test VM %d not found (may not exist yet)", settings.default_vmid)
|
||
|
||
# ISOs
|
||
try:
|
||
raw_isos = await proxmox.get_isos()
|
||
status.available_isos = [
|
||
ISOInfo(
|
||
volid=iso.get("volid", ""),
|
||
filename=iso.get("volid", "").split("/")[-1],
|
||
size=iso.get("size"),
|
||
)
|
||
for iso in raw_isos
|
||
]
|
||
except ProxmoxError:
|
||
logger.debug("Could not list ISOs")
|
||
|
||
status.proxmox_connected = True
|
||
|
||
except ProxmoxError as exc:
|
||
status.error = str(exc)
|
||
logger.warning("Proxmox status check failed: %s", exc)
|
||
except Exception as exc:
|
||
status.error = f"Unexpected error: {exc}"
|
||
logger.exception("Status check failed")
|
||
|
||
# Build queue (always available even if Proxmox is down)
|
||
status.active_builds = get_active_builds()
|
||
|
||
return status
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ISO generation
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.post("/api/generate-iso", response_model=BuildStatus, tags=["ISO"])
|
||
async def generate_iso(config: ISOConfig) -> BuildStatus:
|
||
"""Generate a custom Proxmox auto-install ISO.
|
||
|
||
Accepts the installation configuration, writes an ``answers.toml``,
|
||
and asynchronously runs ``proxmox-auto-install-assistant prepare-iso``.
|
||
Returns the build status immediately; poll ``/api/builds/{id}`` for
|
||
progress.
|
||
"""
|
||
logger.info("ISO generation requested for %s", config.fqdn)
|
||
build = await build_iso(config)
|
||
return build
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Builds
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.get("/api/builds", response_model=list[BuildStatus], tags=["Builds"])
|
||
async def list_builds() -> list[BuildStatus]:
|
||
"""List all ISO builds, newest first."""
|
||
return get_all_builds()
|
||
|
||
|
||
@app.get("/api/builds/{build_id}", response_model=BuildStatus, tags=["Builds"])
|
||
async def get_build_detail(build_id: str) -> BuildStatus:
|
||
"""Get detailed status and logs for a specific build."""
|
||
build = get_build(build_id)
|
||
if build is None:
|
||
raise HTTPException(status_code=404, detail=f"Build {build_id} not found")
|
||
return build
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Deploy
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.post("/api/deploy", response_model=DeployResult, tags=["Deploy"])
|
||
async def deploy_vm(config: DeployConfig) -> DeployResult:
|
||
"""Deploy a VM using the custom auto-install ISO.
|
||
|
||
This will:
|
||
1. Resolve which ISO to use (from *build_id* or the latest build)
|
||
2. Upload the ISO to Proxmox storage if needed
|
||
3. Stop and destroy the existing VM (if any)
|
||
4. Create a new VM with the ISO mounted
|
||
5. Start the VM
|
||
|
||
The entire flow runs synchronously so the caller gets a final status.
|
||
"""
|
||
node = config.node
|
||
vmid = config.vmid
|
||
|
||
# --- Resolve the ISO to deploy ---
|
||
build_id = config.build_id
|
||
if build_id is None:
|
||
latest = get_latest_build()
|
||
if latest is None or latest.state != BuildState.COMPLETED:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="No completed build available. Run /api/generate-iso first.",
|
||
)
|
||
build_id = latest.id
|
||
|
||
build = get_build(build_id)
|
||
if build is None or build.state != BuildState.COMPLETED:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"Build {build_id} is not in a completed state.",
|
||
)
|
||
|
||
iso_path = get_build_iso_path(build_id)
|
||
if iso_path is None or not iso_path.exists():
|
||
raise HTTPException(
|
||
status_code=500,
|
||
detail=f"ISO file for build {build_id} not found on disk.",
|
||
)
|
||
|
||
logger.info(
|
||
"Deploying VM %d on %s from build %s (%s)",
|
||
vmid, node, build_id, iso_path.name,
|
||
)
|
||
|
||
try:
|
||
# --- Upload ISO to Proxmox ---
|
||
iso_volid = f"{config.iso_storage}:iso/{iso_path.name}"
|
||
try:
|
||
existing_isos = await proxmox.get_isos(node, config.iso_storage)
|
||
already_uploaded = any(
|
||
i.get("volid") == iso_volid for i in existing_isos
|
||
)
|
||
except ProxmoxError:
|
||
already_uploaded = False
|
||
|
||
if not already_uploaded:
|
||
logger.info("Uploading ISO %s to Proxmox storage", iso_path.name)
|
||
upload_upid = await proxmox.upload_iso(
|
||
str(iso_path), node, config.iso_storage
|
||
)
|
||
await proxmox.wait_for_task(node, upload_upid, max_wait=300)
|
||
logger.info("ISO upload completed")
|
||
|
||
# --- Tear down existing VM ---
|
||
try:
|
||
vm_status = await proxmox.get_vm_status(node, vmid)
|
||
current_state = vm_status.get("status", "")
|
||
|
||
if current_state == "running":
|
||
logger.info("Stopping existing VM %d", vmid)
|
||
stop_upid = await proxmox.stop_vm(node, vmid)
|
||
await proxmox.wait_for_task(node, stop_upid, max_wait=60)
|
||
# Brief pause for lock release
|
||
await asyncio.sleep(3)
|
||
|
||
logger.info("Destroying existing VM %d", vmid)
|
||
destroy_upid = await proxmox.destroy_vm(node, vmid)
|
||
await proxmox.wait_for_task(node, destroy_upid, max_wait=60)
|
||
await asyncio.sleep(2)
|
||
|
||
except ProxmoxError as exc:
|
||
if exc.status_code == 500 or exc.status_code == 404:
|
||
logger.info("VM %d does not exist, skipping teardown", vmid)
|
||
else:
|
||
raise
|
||
|
||
# --- Create new VM ---
|
||
vm_config: dict[str, Any] = {
|
||
"vmid": vmid,
|
||
"name": build.iso_config.fqdn.split(".")[0] if build.iso_config else f"vm-{vmid}",
|
||
"cores": config.cores,
|
||
"memory": config.memory,
|
||
"ostype": "l26",
|
||
"scsihw": "virtio-scsi-single",
|
||
"scsi0": f"{config.storage}:{config.disk_size.rstrip('G')},iothread=1",
|
||
"ide2": f"{iso_volid},media=cdrom",
|
||
"boot": "order=ide2;scsi0",
|
||
"net0": "virtio,bridge=vmbr0",
|
||
"serial0": "socket",
|
||
"vga": "serial0",
|
||
}
|
||
|
||
create_upid = await proxmox.create_vm(node, vmid, vm_config)
|
||
if isinstance(create_upid, str) and create_upid.startswith("UPID"):
|
||
await proxmox.wait_for_task(node, create_upid, max_wait=30)
|
||
|
||
await asyncio.sleep(1)
|
||
|
||
# --- Start the VM ---
|
||
start_upid = await proxmox.start_vm(node, vmid)
|
||
|
||
return DeployResult(
|
||
vmid=vmid,
|
||
node=node,
|
||
status="started",
|
||
message=f"VM {vmid} created and started with ISO {iso_path.name}",
|
||
task_id=start_upid if isinstance(start_upid, str) else None,
|
||
)
|
||
|
||
except ProxmoxError as exc:
|
||
logger.error("Deploy failed: %s", exc)
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
except Exception as exc:
|
||
logger.exception("Unexpected deploy error")
|
||
raise HTTPException(status_code=500, detail=str(exc))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# VM management
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.get("/api/vm/{vmid}/status", response_model=VMStatus, tags=["VM"])
|
||
async def vm_status(vmid: int, node: str | None = None) -> VMStatus:
|
||
"""Get the current status of a VM."""
|
||
node = node or settings.default_node
|
||
try:
|
||
data = await proxmox.get_vm_status(node, vmid)
|
||
return VMStatus(
|
||
vmid=vmid,
|
||
name=data.get("name"),
|
||
status=data.get("status", "unknown"),
|
||
node=node,
|
||
cpus=data.get("cpus"),
|
||
maxmem=data.get("maxmem"),
|
||
maxdisk=data.get("maxdisk"),
|
||
uptime=data.get("uptime"),
|
||
pid=data.get("pid"),
|
||
)
|
||
except ProxmoxError as exc:
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
|
||
|
||
@app.post("/api/vm/{vmid}/start", tags=["VM"])
|
||
async def vm_start(vmid: int, node: str | None = None) -> dict[str, str]:
|
||
"""Start a VM."""
|
||
node = node or settings.default_node
|
||
try:
|
||
task_id = await proxmox.start_vm(node, vmid)
|
||
return {"status": "starting", "vmid": str(vmid), "task_id": str(task_id)}
|
||
except ProxmoxError as exc:
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
|
||
|
||
@app.post("/api/vm/{vmid}/stop", tags=["VM"])
|
||
async def vm_stop(vmid: int, node: str | None = None) -> dict[str, str]:
|
||
"""Stop a VM."""
|
||
node = node or settings.default_node
|
||
try:
|
||
task_id = await proxmox.stop_vm(node, vmid)
|
||
return {"status": "stopping", "vmid": str(vmid), "task_id": str(task_id)}
|
||
except ProxmoxError as exc:
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ISOs
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.get("/api/isos", response_model=list[ISOInfo], tags=["ISO"])
|
||
async def list_isos(
|
||
node: str | None = None, storage: str | None = None
|
||
) -> list[ISOInfo]:
|
||
"""List available ISO images on Proxmox storage."""
|
||
try:
|
||
raw = await proxmox.get_isos(node, storage)
|
||
return [
|
||
ISOInfo(
|
||
volid=iso.get("volid", ""),
|
||
filename=iso.get("volid", "").split("/")[-1],
|
||
size=iso.get("size"),
|
||
)
|
||
for iso in raw
|
||
]
|
||
except ProxmoxError as exc:
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Static files — mounted last so API routes take precedence
|
||
# ---------------------------------------------------------------------------
|
||
|
||
app.mount("/", StaticFiles(directory=str(_static_dir), html=True), name="static")
|