Merge pull request #163 from theta42/release/v1.23.0-vault-proxy-fix

fix vault 403 for real (v1.23.0)

CHANGELOG.md

@@ -1,3 +1,15 @@
+# v1.23.0
+- fix: /api/vault proxy never injected X-Vault-Token — the true root cause of the recurring vault 403 "permission denied". The proxy declared its hook with http-proxy-middleware v3 syntax (`on: { proxyReq }`), which the installed HPM v2 silently ignores, so every request reached OpenBao unauthenticated (and the client's sso auth headers were never stripped). Rewritten as v2 `onProxyReq`.
+- fix: vault proxy header injection ordered before `fixRequestBody` — the body write flushes headers, so setting X-Vault-Token after it silently failed on every POST/PUT (writes would still 403 even with the hook fixed)
+- fix: initORM add-only schema heal — `sequelize.sync()` never ALTERs existing tables, so columns added by newer releases (e.g. `PluginInstance.lastLog`, which crashed the scheduler on every boot of an upgraded deployment) are now detected via describeTable and added with addColumn (additive only, per-column fail-soft)
+- feat: external-app vault tokens are long-lived and auto-renewed — minted via the new `sso-app` token role (periodic 768h, falls back to sso-broker's 24h role until theta-suite setup.sh is re-run); sso stores each token's accessor (new VaultAppToken model — an accessor can renew/revoke but not authenticate) and renews all of them at boot + every 6h via auth/token/renew-accessor, so a downstream app's credential stays valid as long as sso runs with zero renewal code in the app
+- feat: re-minting an app token revokes the app's previous token via its stored accessor — exactly one live credential per app, no zombies
+- test: wire-level tests for the vault proxy (real HTTP round-trip asserting token injection, auth-header stripping, path rewrite, and POST body integrity) + app-token accessor lifecycle tests
+
+# v1.22.0
+- feat: Agents page — live list of connected theta-agent hosts with telemetry (CPU/RAM/disk/ZFS/GPU) + online status, updating via socket.io
+- security: auth + admin-gate the /api/agent REST routes (previously unauthenticated)
+
 # v1.21.0
 - fix: always reconcile OpenBao policy content before serving a (possibly cached) token, so stale stored policies can no longer cause a recurring vault 403 "permission denied"
 - feat: shared secrets — users can publish secrets to secret/shared/<owner>/<slug> and grant read access to other users and downstream apps (OpenBao ACL policy edits, applied live)

nodejs/bin/www

@@ -60,6 +60,13 @@ models.initORM().then(() => {
   initScheduler(conf.discovery).catch(err => {
     console.error('Failed to initialize scheduler:', err);
   });
+
+  // Keep external-app vault tokens alive: renew every stored accessor now and
+  // on an interval (see vault_broker.startAppTokenRenewal). Only meaningful
+  // when OpenBao is configured; without VAULT_TOKEN the loop's calls fail soft.
+  if (process.env.VAULT_TOKEN) {
+    require('../utils/vault_broker').startAppTokenRenewal();
+  }
 }).catch(err => {
   console.error('Failed to initialize ORM:', err);
   process.exit(1);

nodejs/models/index.js

@@ -19,6 +19,7 @@ const { Webhook } = require('./webhook');
 const { PluginInstance } = require('./plugin_instance');
 const { SharedSecret } = require('./shared_secret');
 const { SharedSecretGrant } = require('./shared_secret_grant');
+const { VaultAppToken } = require('./vault_app_token');
 async function initORM() {
   const ormConf = conf.orm || {
     dialect: 'sqlite',
@@ -33,16 +34,48 @@ async function initORM() {
       conf: { orm: ormConf },
       models: [
         Resource, ResourceEdge, ResourceGroup, AccessRequest, Webhook, PluginInstance,
-        SharedSecret, SharedSecretGrant,
+        SharedSecret, SharedSecretGrant, VaultAppToken,
         Token, AuthToken, InviteToken, ImpersonationToken, PasswordResetToken, OtpToken, ServiceToken
       ]
     });
     console.log('[initORM] ORM initialized successfully');
     console.log('[initORM] Resource.orm =', !!Resource.orm, 'Token.orm =', !!Token.orm);
+    await healSchema();
   } catch (err) {
     console.error('[initORM] ORM initialization failed:', err.message);
     throw err;
   }
 }
 
+// Add-only schema heal. @simpleworkjs/orm runs sequelize.sync() WITHOUT alter,
+// which creates missing tables but never touches existing ones — so a column
+// added in a newer release (e.g. PluginInstance.lastLog) simply never appears
+// in an upgraded deployment's database and every query on the model fails
+// ("no such column"). This walks each Sequelize model and ADDs any attribute
+// missing from its table. Strictly additive (never drops or retypes), works on
+// any dialect via the query interface, and fail-soft per column so one bad
+// attribute can't take the boot down.
+async function healSchema() {
+  const adapter = Resource.orm && Resource.orm.adapters && Resource.orm.adapters.sequelize;
+  if (!adapter || !adapter.sequelize) return;
+  const sequelize = adapter.sequelize;
+  const qi = sequelize.getQueryInterface();
+  for (const SM of Object.values(sequelize.models)) {
+    const table = SM.getTableName();
+    let existing;
+    try { existing = await qi.describeTable(table); }
+    catch (e) { continue; } // no table yet — sync() handles creation
+    for (const [name, attr] of Object.entries(SM.getAttributes())) {
+      const col = attr.field || name;
+      if (existing[col]) continue;
+      try {
+        await qi.addColumn(table, col, attr);
+        console.log(`[initORM] schema heal: added missing column ${table}.${col}`);
+      } catch (e) {
+        console.error(`[initORM] schema heal: could not add ${table}.${col}:`, e.message);
+      }
+    }
+  }
+}
+
 module.exports.initORM = initORM;

nodejs/models/vault_app_token.js

@@ -0,0 +1,43 @@
+'use strict';
+
+// VaultAppToken — the ACCESSOR of an OpenBao token minted for an external app
+// from the vault UI (Apps tab), so sso can keep the token alive.
+//
+// The token itself is shown ONCE at mint and never stored (a stolen accessor
+// cannot authenticate — it can only look up, renew, or revoke its token, and
+// only the sso broker's policy grants those endpoints). App tokens are minted
+// through the sso-app role as PERIODIC tokens: they live forever, but only if
+// something renews them inside every period window. That something is sso's
+// renewal loop (vault_broker.startAppTokenRenewal), which walks these rows and
+// POSTs auth/token/renew-accessor on a timer — so a downstream app's credential
+// stays valid as long as sso itself is running, with no renewal code needed in
+// the downstream app.
+//
+// One row per app name: re-minting an app's token revokes the previous token
+// via its accessor (no zombie credentials) and replaces the row.
+
+const { Model } = require('@simpleworkjs/orm');
+
+class VaultAppToken extends Model {
+  static fields = {
+    id: { type: 'uuid', primaryKey: true },
+    // The external app's name — also its policy (app-<name>) and KV namespace
+    // (secret/apps/<name>/). Unique: one live token per app.
+    name: { type: 'string', isRequired: true, unique: true, min: 1, max: 64 },
+    // The minted token's accessor (renew/revoke handle, cannot authenticate).
+    accessor: { type: 'string', isRequired: true, max: 128 },
+    // Renewal bookkeeping, updated by the renewal loop.
+    lastRenewedAt: { type: 'integer' },
+    lastError: { type: 'text' },
+    // Audit stamps (set by the route handler, not by an ORM hook).
+    created_by: { type: 'string' },
+    created_on: { type: 'integer' },
+  };
+
+  static async getByName(name) {
+    const rows = await this.list({ where: { name } });
+    return rows[0] || null;
+  }
+}
+
+module.exports = { VaultAppToken };

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "t42-sso-manager",
-  "version": "1.21.0",
+  "version": "1.23.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "t42-sso-manager",
-      "version": "1.21.0",
+      "version": "1.23.0",
       "license": "MIT",
       "dependencies": {
         "@fortawesome/fontawesome-free": "^7.3.0",

nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "t42-sso-manager",
-  "version": "1.21.0",
+  "version": "1.23.0",
   "description": "A very simple LDAP management and SSO system",
   "author": [
     {

nodejs/routes/api_agent.js

@@ -1,8 +1,12 @@
 'use strict';
 
 const express = require('express');
+const middleware = require('../middleware/auth');
+const permission = require('../utils/permission');
 const agentManager = require('../utils/agent_manager');
 
+const ADMIN_GROUPS = ['app_sso_admin', 'app_super_admin', 'app_sso_directory_admin'];
+
 module.exports = function initAgentWebSockets(app) {
   if (!app.wss) {
     console.warn("WebSocket server for agents is not initialized.");
@@ -71,8 +75,23 @@ module.exports = function initAgentWebSockets(app) {
     } catch (e) {}
   });
 
-  // REST API routes for Agent Management (mounted under /api/agent)
+  // REST API routes for Agent Management (mounted under /api/agent). The agent
+  // WebSocket (/api/agent/ws) is handled by the raw `wss` upgrade server in
+  // bin/www with its own ?token= auth — unaffected by the express middleware
+  // here. These REST routes are admin-facing, so they're auth + admin gated.
   const router = express.Router();
+  router.use(middleware.auth);
+  router.use(async (req, res, next) => {
+    try {
+      await permission.byGroup(req.user, ADMIN_GROUPS);
+      next();
+    } catch (err) {
+      if (err && (err.status === 401 || err.name === 'Insufficient Permission')) {
+        return res.status(403).json({ status: 'error', message: 'admin only' });
+      }
+      next(err);
+    }
+  });
 
   router.get('/nodes', (req, res) => {
     res.json({

nodejs/routes/index.js

@@ -59,6 +59,12 @@ router.get('/overview', function(req, res) {
   res.render('overview', {...values});
 });
 
+// Connected theta-agent hosts + live telemetry (admin). Data from
+// GET /api/agent/nodes; live updates via socket.io 'agent.*' events.
+router.get('/agents', function(req, res) {
+  res.render('agents', {...values});
+});
+
 router.get('/admin', (req, res) => res.redirect(301, '/overview'));
 router.get('/notifications', (req, res) => res.redirect(301, '/overview'));
 router.get('/dashboard', (req, res) => res.redirect(301, '/overview'));

nodejs/tests/vault_broker.test.js

@@ -15,8 +15,36 @@ jest.mock('redis', () => ({
   })
 }));
 
+// In-memory stand-ins for the ORM-backed models so mintAppToken/renewAppTokens
+// can run without a database.
+jest.mock('../models/shared_secret', () => ({
+  SharedSecret: { list: jest.fn().mockResolvedValue([]) },
+}));
+jest.mock('../models/shared_secret_grant', () => ({
+  SharedSecretGrant: { listForGrantee: jest.fn().mockResolvedValue([]) },
+}));
+jest.mock('../models/vault_app_token', () => {
+  const rows = [];
+  const VaultAppToken = {
+    _rows: rows,
+    list: jest.fn(async () => rows),
+    getByName: jest.fn(async (name) => rows.find(r => r.name === name) || null),
+    create: jest.fn(async (data) => {
+      const row = {
+        ...data,
+        update: jest.fn(async function (patch) { Object.assign(this, patch); }),
+        delete: jest.fn(async function () { rows.splice(rows.indexOf(this), 1); }),
+      };
+      rows.push(row);
+      return row;
+    }),
+  };
+  return { VaultAppToken };
+});
+
 const baoConf = require('@simpleworkjs/bao-conf');
 const vaultBroker = require('../utils/vault_broker');
+const { VaultAppToken } = require('../models/vault_app_token');
 
 describe('vault_broker admin policy', () => {
   beforeEach(() => {
@@ -49,3 +77,128 @@ describe('vault_broker admin policy', () => {
     }));
   });
 });
+
+describe('app token lifecycle (accessor storage + renewal)', () => {
+  beforeEach(() => {
+    baoConf.request.mockReset();
+    VaultAppToken._rows.length = 0;
+  });
+
+  function mockBao({ mintAccessor = 'acc-1', renewOk = true } = {}) {
+    baoConf.request.mockImplementation(async (method, path, body) => {
+      if (path.startsWith('sys/policies/acl/')) {
+        if (method === 'GET') return { status: 404, text: async () => '' };
+        return { status: 204, ok: true };
+      }
+      if (path === 'auth/token/create/sso-app') {
+        return { ok: true, json: async () => ({ auth: { client_token: 'app-tok', accessor: mintAccessor, lease_duration: 2764800 } }) };
+      }
+      if (path === 'auth/token/renew-accessor') {
+        return renewOk ? { ok: true, json: async () => ({}) } : { ok: false, status: 400, text: async () => 'invalid accessor' };
+      }
+      if (path === 'auth/token/revoke-accessor') {
+        return { ok: true, status: 204, text: async () => '' };
+      }
+      return { status: 200, ok: true, json: async () => ({}) };
+    });
+  }
+
+  test('mintAppToken stores the accessor; re-mint revokes the old accessor and replaces the row', async () => {
+    mockBao({ mintAccessor: 'acc-old' });
+    await vaultBroker.mintAppToken('demo', 'adminuser');
+    expect(VaultAppToken._rows).toHaveLength(1);
+    expect(VaultAppToken._rows[0]).toMatchObject({ name: 'demo', accessor: 'acc-old', created_by: 'adminuser' });
+
+    mockBao({ mintAccessor: 'acc-new' });
+    await vaultBroker.mintAppToken('demo', 'adminuser');
+    expect(baoConf.request).toHaveBeenCalledWith('POST', 'auth/token/revoke-accessor', { accessor: 'acc-old' });
+    expect(VaultAppToken._rows).toHaveLength(1);
+    expect(VaultAppToken._rows[0].accessor).toBe('acc-new');
+  });
+
+  test('renewAppTokens renews each accessor and stamps lastRenewedAt', async () => {
+    mockBao();
+    await vaultBroker.mintAppToken('demo', 'adminuser');
+    VaultAppToken._rows[0].lastRenewedAt = 0;
+    await vaultBroker.renewAppTokens();
+    expect(baoConf.request).toHaveBeenCalledWith('POST', 'auth/token/renew-accessor', { accessor: 'acc-1' });
+    expect(VaultAppToken._rows[0].lastRenewedAt).toBeGreaterThan(0);
+    expect(VaultAppToken._rows[0].lastError).toBeNull();
+  });
+
+  test('renewAppTokens records the failure on the row without throwing', async () => {
+    mockBao({ renewOk: false });
+    await vaultBroker.mintAppToken('demo', 'adminuser');
+    await vaultBroker.renewAppTokens();
+    expect(VaultAppToken._rows[0].lastError).toMatch(/renew failed \(400\)/);
+  });
+});
+
+// Real HTTP round-trip through vaultProxy() against an in-process fake OpenBao.
+// This exists because the proxy once shipped with a hook shape the installed
+// http-proxy-middleware version ignored (v3 `on: { proxyReq }` vs v2
+// `onProxyReq`), so NO X-Vault-Token was ever injected and every /api/vault
+// request 403'd. A unit test on options can't catch that — only a wire test can.
+describe('vaultProxy wire behavior', () => {
+  const http = require('http');
+  const express = require('express');
+
+  let target; // fake OpenBao
+  let seen;   // last request the fake OpenBao received
+  let app;    // sso app fragment: scopeGuard stub + vaultProxy
+  let server;
+
+  beforeAll((done) => {
+    target = http.createServer((req, res) => {
+      let body = '';
+      req.on('data', (c) => { body += c; });
+      req.on('end', () => {
+        seen = { method: req.method, url: req.url, headers: req.headers, body };
+        res.setHeader('content-type', 'application/json');
+        res.end('{"ok":true}');
+      });
+    });
+    target.listen(0, '127.0.0.1', () => {
+      process.env.VAULT_ADDR = `http://127.0.0.1:${target.address().port}`;
+      jest.resetModules();
+      const broker = require('../utils/vault_broker');
+      app = express();
+      app.use(express.json());
+      app.use('/api/vault', (req, res, next) => { req.vaultToken = 'scoped-token-123'; next(); }, broker.vaultProxy());
+      server = app.listen(0, '127.0.0.1', done);
+    });
+  });
+
+  afterAll((done) => {
+    server.close(() => target.close(done));
+  });
+
+  function call(path, opts = {}) {
+    const port = server.address().port;
+    return fetch(`http://127.0.0.1:${port}${path}`, opts);
+  }
+
+  test('GET list rewrites /api/vault -> /v1, injects X-Vault-Token, strips sso auth headers', async () => {
+    const res = await call('/api/vault/secret/metadata/users/alice?list=true', {
+      headers: { 'auth-token': 'sso-session-token', authorization: 'Bearer sso_x_y', 'content-type': 'application/json' },
+    });
+    expect(res.status).toBe(200);
+    expect(seen.url).toBe('/v1/secret/metadata/users/alice?list=true');
+    expect(seen.headers['x-vault-token']).toBe('scoped-token-123');
+    expect(seen.headers['auth-token']).toBeUndefined();
+    expect(seen.headers['authorization']).toBeUndefined();
+  });
+
+  test('POST body survives the express.json + fixRequestBody round-trip', async () => {
+    const res = await call('/api/vault/secret/data/users/alice/foo', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', 'auth-token': 'sso-session-token' },
+      body: JSON.stringify({ data: { hello: 'world' } }),
+    });
+    expect(res.status).toBe(200);
+    expect(seen.method).toBe('POST');
+    expect(seen.url).toBe('/v1/secret/data/users/alice/foo');
+    expect(seen.headers['x-vault-token']).toBe('scoped-token-123');
+    expect(JSON.parse(seen.body)).toEqual({ data: { hello: 'world' } });
+  });
+});

nodejs/utils/ui.js

@@ -46,6 +46,7 @@ module.exports = {
 		{href: '/directory', icon: 'fa-solid fa-server', label: 'Directory', groups: ['app_sso_admin', 'app_sso_directory_admin', 'admin']},
 		// Vault requires login - per-user secrets at secret/users/<uid>/*.
 		{href: '/vault', icon: 'fa-solid fa-vault', label: 'Vault', groups: ['login']},
+		{href: '/agents', icon: 'fa-solid fa-microchip', label: 'Agents', groups: ['app_sso_admin', 'admin']},
 		{href: '/overview', icon: 'fa-solid fa-gauge-high', label: 'Overview', groups: ['app_sso_admin', 'admin']},
 	],
 };

nodejs/utils/vault_broker.js

@@ -32,6 +32,7 @@ const conf = require('@simpleworkjs/conf');
 const permission = require('./permission');
 const { SharedSecret } = require('../models/shared_secret');
 const { SharedSecretGrant } = require('../models/shared_secret_grant');
+const { VaultAppToken } = require('../models/vault_app_token');
 
 const ROLE = 'sso-broker';
 const DEFAULT_TTL = 24 * 60 * 60; // matches the role's token_period (24h)
@@ -83,15 +84,18 @@ async function ensurePolicy(name, hcl) {
 	await bao('PUT', `sys/policies/acl/${name}`, { policy: hcl });
 }
 
-// Mint a token through the sso-broker role with the given policies. Returns
-// { token, ttl } (ttl = lease_duration seconds, falls back to DEFAULT_TTL).
-async function mintToken(policies) {
-	const res = await bao('POST', 'auth/token/create/sso-broker', { policies });
+// Mint a token through a token role with the given policies. Returns
+// { token, accessor, ttl } (ttl = lease_duration seconds, falls back to
+// DEFAULT_TTL). Roles: sso-broker (24h period — user/admin tokens, re-minted
+// from cache) and sso-app (768h period — long-lived external-app credentials,
+// kept alive via their stored accessor by the renewal loop below).
+async function mintToken(policies, role = ROLE) {
+	const res = await bao('POST', `auth/token/create/${role}`, { policies });
 	const json = await res.json();
 	const token = json && json.auth && json.auth.client_token;
 	if (!token) throw new Error(`OpenBao token mint returned no client_token: ${JSON.stringify(json)}`);
 	const ttl = (json.auth && json.auth.lease_duration) || DEFAULT_TTL;
-	return { token, ttl };
+	return { token, accessor: json.auth.accessor, ttl };
 }
 
 // ── Shared-secret policy rules ───────────────────────────────────────────────
@@ -185,15 +189,94 @@ ${granted}`.trim();
 // a later compromise of an admin session cannot recover previously-minted app
 // tokens. The caller must record it in the external app immediately. Later
 // grants to the app edit app-<name> policy content (live-applied to this token).
-async function mintAppToken(name) {
+//
+// What IS stored is the token's ACCESSOR (VaultAppToken row): an accessor
+// cannot authenticate, but it lets the renewal loop below keep the (periodic)
+// token alive and lets a re-mint revoke the app's previous token so exactly
+// one credential per app is ever live.
+async function mintAppToken(name, actorUid) {
 	if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(name)) {
 		throw new Error('invalid app name (lowercase letters, digits, hyphens; max 63 chars)');
 	}
 	await ensurePolicy(`app-${name}`, await appPolicyHcl(name));
-	const { token, ttl } = await mintToken([`app-${name}`]);
+	// App tokens are long-lived credentials: mint via the sso-app role (768h
+	// period) so a renewal inside every 32-day window keeps them alive forever.
+	// Fall back to the broker's own 24h role on deployments whose setup.sh
+	// predates the sso-app role (re-running setup.sh creates it).
+	let minted;
+	try {
+		minted = await mintToken([`app-${name}`], 'sso-app');
+	} catch (e) {
+		console.warn(`vault_broker: sso-app token role unavailable (${e.message}); falling back to sso-broker (24h period). Re-run theta-env setup.sh to create the sso-app role.`);
+		minted = await mintToken([`app-${name}`]);
+	}
+	const { token, accessor, ttl } = minted;
+	// Replace the app's accessor row; revoke the superseded token (best-effort —
+	// it may already be expired) so re-minting never leaves a zombie credential.
+	try {
+		const existing = await VaultAppToken.getByName(name);
+		if (existing) {
+			await baoConf.request('POST', 'auth/token/revoke-accessor', { accessor: existing.accessor });
+			await existing.delete();
+		}
+		if (accessor) {
+			await VaultAppToken.create({
+				name, accessor,
+				lastRenewedAt: Date.now(),
+				created_by: actorUid, created_on: Date.now(),
+			});
+		}
+	} catch (e) {
+		// Accessor bookkeeping must never block handing the token out; without a
+		// row the token simply isn't auto-renewed (it still lives one full period).
+		console.error(`vault_broker: could not store accessor for app-${name}:`, e.message);
+	}
 	return { token, ttl, policy: `app-${name}`, path: `secret/apps/${name}/` };
 }
 
+// ── App-token renewal loop ──────────────────────────────────────────────────
+// Walks the stored accessors and renews each token (auth/token/renew-accessor),
+// resetting its periodic clock. Runs at boot and then every RENEW_INTERVAL_MS —
+// far inside both possible periods (24h fallback and 768h), so a downstream
+// app's token stays valid for as long as sso is running. Failures are recorded
+// on the row (visible to admins in the DB / future UI) and never throw.
+const RENEW_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6h — several chances per 24h period
+let renewTimer;
+
+async function renewAppTokens() {
+	let rows;
+	try { rows = await VaultAppToken.list(); }
+	catch (e) { console.error('vault_broker: app-token renewal: could not list accessors:', e.message); return; }
+	for (const row of rows) {
+		try {
+			const res = await baoConf.request('POST', 'auth/token/renew-accessor', { accessor: row.accessor });
+			if (res.ok) {
+				await row.update({ lastRenewedAt: Date.now(), lastError: null });
+			} else {
+				const text = await res.text().catch(() => '');
+				// 400 "invalid accessor" = token expired or was revoked out-of-band;
+				// keep the row + error so the admin can see the app needs a re-mint.
+				await row.update({ lastError: `renew failed (${res.status}) ${text}` });
+				console.warn(`vault_broker: renew of app token '${row.name}' failed (${res.status}) — re-mint it from the vault UI if the app is still in use.`);
+			}
+		} catch (e) {
+			try { await row.update({ lastError: e.message }); } catch (e2) { /* best-effort */ }
+			console.error(`vault_broker: renew of app token '${row.name}' errored:`, e.message);
+		}
+	}
+}
+
+// Start the loop (idempotent). unref() so an open handle never blocks exit.
+function startAppTokenRenewal() {
+	if (renewTimer) return renewTimer;
+	renewAppTokens().catch((e) => console.error('vault_broker: initial app-token renewal failed:', e.message));
+	renewTimer = setInterval(() => {
+		renewAppTokens().catch((e) => console.error('vault_broker: app-token renewal failed:', e.message));
+	}, RENEW_INTERVAL_MS);
+	if (renewTimer.unref) renewTimer.unref();
+	return renewTimer;
+}
+
 // ── Grant / revoke shared-secret access ─────────────────────────────────────
 // Creating a grant writes the DB row and then edits the grantee's policy content
 // to add read on the shared path; revoking removes both. Because OpenBao parses
@@ -292,15 +375,20 @@ function vaultProxy() {
 		target: VAULT_ADDR,
 		changeOrigin: true,
 		pathRewrite: { '^/api/vault': '/v1' },
-		on: {
-			proxyReq(proxyReq, req, res, options) {
-				fixRequestBody(proxyReq, req, res, options);
-				// Inject ONLY the server-minted scoped token; strip the client's
-				// sso session/api auth so it never reaches OpenBao.
-				proxyReq.setHeader('X-Vault-Token', req.vaultToken);
-				proxyReq.removeHeader('auth-token');
-				proxyReq.removeHeader('authorization');
-			},
+		// http-proxy-middleware v2 API: hooks are top-level onProxyReq/onError,
+		// NOT the v3 `on: { proxyReq }` shape. v2 silently ignores an `on` key,
+		// which shipped this proxy with NO token injection — every /api/vault
+		// call reached OpenBao unauthenticated and 403'd.
+		onProxyReq(proxyReq, req, res, options) {
+			// Header ops MUST precede fixRequestBody: it write()s the parsed body
+			// onto proxyReq, which flushes headers — setHeader after that throws
+			// (swallowed upstream), silently dropping the token on every write.
+			// Inject ONLY the server-minted scoped token; strip the client's
+			// sso session/api auth so it never reaches OpenBao.
+			proxyReq.setHeader('X-Vault-Token', req.vaultToken);
+			proxyReq.removeHeader('auth-token');
+			proxyReq.removeHeader('authorization');
+			fixRequestBody(proxyReq, req, res, options);
 		},
 	});
 }
@@ -314,7 +402,7 @@ mintAppRouter.post('/', async (req, res, next) => {
 		await permission.byGroup(req.user, [ADMIN_GROUP]);
 		const name = (req.body && req.body.name || '').trim();
 		if (!name) return res.status(400).json({ error: 'name is required' });
-		const result = await mintAppToken(name);
+		const result = await mintAppToken(name, req.user && req.user.uid);
 		res.json(result);
 	} catch (e) {
 		if (e.status === 401) return res.status(403).json({ error: 'admin only' });
@@ -330,6 +418,10 @@ module.exports = {
 	scopeGuard,
 	vaultProxy,
 	mintAppRouter,
+	// app-token lifecycle
+	renewAppTokens,
+	startAppTokenRenewal,
+	VaultAppToken,
 	// sharing
 	SharedSecret,
 	SharedSecretGrant,

nodejs/views/agents.ejs

@@ -0,0 +1,112 @@
+<%- include('top') %>
+
+<div class="container-fluid py-4">
+  <div class="d-flex justify-content-between align-items-center mb-3">
+    <h2><i class="fa-solid fa-microchip"></i> Theta Agents <small class="text-muted">(connected hosts)</small></h2>
+    <button class="btn btn-outline-primary" onclick="loadAgents()"><i class="fa-solid fa-rotate"></i> Refresh</button>
+  </div>
+
+  <div class="card shadow-sm">
+    <div class="card-header bg-light"><h5 class="card-title mb-0">Connected agents</h5></div>
+    <div class="table-responsive">
+      <table class="table table-hover align-middle mb-0">
+        <thead class="table-light">
+          <tr>
+            <th>Host</th>
+            <th>IP</th>
+            <th>Status</th>
+            <th style="width:110px">CPU</th>
+            <th style="width:110px">RAM</th>
+            <th style="width:110px">Disk</th>
+            <th>ZFS</th>
+            <th>GPU</th>
+            <th>Last seen</th>
+          </tr>
+        </thead>
+        <tbody id="agents-tbody">
+          <tr><td colspan="9" class="text-center text-muted">Loading agents...</td></tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+
+  <p class="text-muted small mt-3">
+    Live data from the theta-agent telemetry stream. An agent reports hostname/IP discovery and
+    CPU/RAM/disk/ZFS/GPU usage every ~60s over the WebSocket; "Online" means seen in the last 90s.
+  </p>
+</div>
+
+<script type="text/javascript">
+  app.auth.forceLogin(['app_sso_admin', 'admin']);
+
+  let agentsById = {}; // token -> agent record
+
+  function bar(val) {
+    val = Math.max(0, Math.min(100, val || 0));
+    return `<div class="progress" style="height:8px"><div class="progress-bar" role="progressbar" style="width:${val}%"></div></div>`;
+  }
+  function timeAgo(iso) {
+    if (!iso) return '';
+    const m = moment(iso);
+    return m.isValid() ? m.fromNow() : '';
+  }
+  function esc(s) {
+    if (s == null) return '';
+    return app.util.escapeHtml(String(s));
+  }
+
+  function renderRow(id, a) {
+    const d = a.discovery || {};
+    const t = a.telemetry || {};
+    const online = !!a.isOnline;
+    const badge = `<span class="badge ${online ? 'bg-success' : 'bg-secondary'}">${online ? 'Online' : 'Offline'}</span>`;
+    const cpu = t.cpu_usage_percent != null ? t.cpu_usage_percent : 0;
+    const ram = t.ram_usage_percent != null ? t.ram_usage_percent : 0;
+    const disk = t.disk_usage_percent != null ? t.disk_usage_percent : 0;
+    const gpu = (t.gpu_usage_percent != null && t.gpu_usage_percent >= 0) ? t.gpu_usage_percent + '%' : 'N/A';
+    return `<tr id="agent-${id}">
+      <td><strong>${esc(a.hostname || 'unknown')}</strong>${d.location ? `<div class="small text-muted">${esc(d.location)}</div>` : ''}</td>
+      <td>${esc(a.ipAddress || '')}</td>
+      <td>${badge}</td>
+      <td>${cpu}% ${bar(cpu)}</td>
+      <td>${ram}% ${bar(ram)}</td>
+      <td>${disk}% ${bar(disk)}</td>
+      <td>${esc(t.zfs_health || 'N/A')}</td>
+      <td>${gpu}</td>
+      <td class="small text-muted">${timeAgo(a.lastSeen)}</td>
+    </tr>`;
+  }
+
+  async function loadAgents() {
+    const tbody = document.getElementById('agents-tbody');
+    try {
+      const res = await app.api.get('agent/nodes');
+      const agents = (res && res.agents) || [];
+      agentsById = {};
+      agents.forEach(a => { agentsById[a.token] = a; });
+      tbody.innerHTML = agents.length
+        ? agents.map(a => renderRow(a.token, a)).join('')
+        : '<tr><td colspan="9" class="text-center text-muted">No agents connected.</td></tr>';
+    } catch (err) {
+      tbody.innerHTML = `<tr><td colspan="9" class="text-center text-danger">Error loading agents: ${esc(err && err.message || err)}</td></tr>`;
+    }
+  }
+
+  // Live updates from the server's agent.* socket.io broadcasts. The app's
+  // default socket is scoped to P2PSub, so open a dedicated socket here.
+  const agentSocket = io({ auth: { token: app.auth.getToken() } });
+  agentSocket.on('agent.telemetry', (msg) => {
+    const a = agentsById[msg && msg.token];
+    if (a) { a.telemetry = msg.payload; a.isOnline = true; const row = document.getElementById('agent-' + msg.token); if (row) row.outerHTML = renderRow(msg.token, a); }
+  });
+  agentSocket.on('agent.discovery', (msg) => {
+    const a = agentsById[msg && msg.token];
+    if (a) { a.discovery = msg.payload; a.hostname = (msg.payload && msg.payload.hostname) || a.hostname; const row = document.getElementById('agent-' + msg.token); if (row) row.outerHTML = renderRow(msg.token, a); }
+  });
+
+  loadAgents();
+  // Re-fetch periodically to reflect connect/disconnect + isOnline (90s window).
+  setInterval(loadAgents, 30000);
+</script>
+
+<%- include('bottom') %>

nodejs/views/vault.ejs

@@ -56,6 +56,7 @@
             <div class="card-header bg-light"><h5 class="card-title mb-0">Mint an app token</h5></div>
             <div class="card-body">
               <p class="text-muted small">Mints a scoped OpenBao token confined to <code>secret/apps/&lt;name&gt;/*</code> for an external app. The token is shown <strong>once</strong> — record it in the app immediately; it cannot be recovered later.</p>
+              <p class="text-muted small">The token is periodic: it stays valid as long as the app renews it within its period (<code>POST /v1/auth/token/renew-self</code>). If it lapses, mint a new one here — the app's policy and stored secrets are kept.</p>
               <div class="mb-3">
                 <label class="form-label">App name (lowercase letters, digits, hyphens)</label>
                 <input type="text" class="form-control" id="app-name-input" placeholder="e.g. my-service">