feat: real plugin system with loadable instances + OpenBao secrets (v1.17.0)

Generalize the half-built discovery plugins into a real plugin system: plugin
TYPES (the plugins/<category>/<type>.js modules with manifests) and loadable,
configurable, multi-copy plugin INSTANCES (PluginInstance ORM model) managed
from a dedicated /plugins page and /api/plugins API, with per-instance secrets
in OpenBao at secret/plugins/<id>/conf.

- plugin_registry.js: getTypes/getModule/splitConfig/mask + required-field helpers
- PluginInstance model (Sequelize): id/pluginType/category/name/slug(unique)/
  enabled/cron/config(json, non-secret)/lastRun*; registered in models/index.js
- plugin_secrets.js: read/write/remove/mergeForRun over @simpleworkjs/bao-conf
- scheduler.js: schedules from the DB registry; per-instance stable BullMQ
  JobScheduler ids (plugin:<id>) for load/unload; legacy migration from
  conf.discovery.plugins on first boot (idempotent, empty-table-guarded)
- api_plugins.js (replaces routes/plugins.js): types/list/get/create/update/
  secrets/test/load/unload/run/delete/runs; admin-gated; secrets always masked
- /plugins page (plugins.ejs) + nav; Agents & Scheduler tab removed from
  /directory; /docs/agents aliased to /docs/plugins
- proxmox/unifi/nmap gained manifests (configSchema/validate/run alias)
- tests/plugins.test.js: registry unit + plugin_secrets (mocked bao-conf) +
  PluginInstance model round-trip/unique-slug
- docs (plugins.md, vault.md, _config.yml, API.md) + 1.16.1 -> 1.17.0

Requires theta-suite >= v1.30.1 for the sso-broker secret/plugins/* grant;
fails-soft with a clear error if absent.

Co-Authored-By: Claude <noreply@anthropic.com>

nodejs/routes/api_plugins.js

@@ -0,0 +1,262 @@
+'use strict';
+
+// Plugin instances API — the loadable, configurable, multi-copy plugin system.
+//
+// Replaces the old routes/plugins.js (which only toggled cron/enabled on static
+// config via a Redis hash). Here every plugin is a PluginInstance row (see
+// models/plugin_instance.js) with its own schedule and its secrets in OpenBao
+// (utils/plugin_secrets.js), created/edited/loaded/unloaded through this API.
+//
+// Gated router-wide to the same admin groups as the directory admin API, so
+// existing directory admins keep access. Secrets are never returned in
+// cleartext — only masked (`********`) — and never persisted in the DB.
+
+const router = require('express').Router();
+const permission = require('../utils/permission');
+const registry = require('../services/plugin_registry');
+const pluginSecrets = require('../utils/plugin_secrets');
+const { PluginInstance, STATUS } = require('../models/plugin_instance');
+const { scheduleInstance, unscheduleInstance, runInstanceNow } = require('../services/scheduler');
+
+const SLUG_RE = /^[a-z0-9][a-z0-9_-]{0,63}$/;
+
+// Same gate as the directory admin API: app_sso_admin or app_sso_directory_admin
+// (app_super_admin is always allowed by permission.byGroup).
+router.use(async (req, res, next) => {
+  try {
+    await permission.byGroup(req.user, ['app_sso_directory_admin', 'app_sso_admin']);
+    next();
+  } catch (err) { next(err); }
+});
+
+// Plain object for the wire, with masked secret values attached under
+// `secrets` and the run-state fields surfaced. The DB row never holds secrets.
+async function serialize(instance) {
+  const obj = instance.toJSON ? instance.toJSON() : { ...instance };
+  const secrets = await pluginSecrets.read(instance.id).catch(() => ({}));
+  obj.secrets = registry.mask(instance.pluginType, secrets);
+  return obj;
+}
+
+// Validate a create/update payload against a plugin type's configSchema.
+// Returns an error string or null. `flat` is the merged config + secret values
+// (the UI sends one flat object; the API splits it).
+function validateFields(type, flat) {
+  const required = registry.requiredKeys(type);
+  for (const key of required) {
+    const v = flat && flat[key];
+    if (v === undefined || v === null || v === '') {
+      return `Missing required field: ${key}`;
+    }
+  }
+  return null;
+}
+
+// --- Plugin types (for the create-instance picker + form) ---
+router.get('/types', (req, res) => {
+  res.json({ results: registry.getTypes() });
+});
+
+// --- List instances ---
+router.get('/', async (req, res, next) => {
+  try {
+    const instances = await PluginInstance.list();
+    const out = [];
+    for (const inst of instances) out.push(await serialize(inst));
+    res.json({ results: out });
+  } catch (err) { next(err); }
+});
+
+router.get('/:id', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    res.json({ results: await serialize(inst) });
+  } catch (err) { next(err); }
+});
+
+// --- Create instance ---
+router.post('/', async (req, res, next) => {
+  try {
+    const { pluginType, name, slug, cron } = req.body;
+    if (!pluginType) return res.status(400).json({ error: 'pluginType is required' });
+    if (!registry.getManifest(pluginType)) return res.status(400).json({ error: `Unknown plugin type: ${pluginType}` });
+    if (!name) return res.status(400).json({ error: 'name is required' });
+    if (!slug || !SLUG_RE.test(slug)) return res.status(400).json({ error: 'slug must be lowercase letters/digits/_/- (max 64)' });
+    if (cron !== undefined && (typeof cron !== 'string' || !cron.trim())) return res.status(400).json({ error: 'cron must be a non-empty string' });
+
+    // `config` from the client is a flat object of all field values (secret +
+    // non-secret). Split it: non-secret -> DB, secret -> OpenBao.
+    const flat = (req.body.config && typeof req.body.config === 'object') ? req.body.config : {};
+    const fieldErr = validateFields(pluginType, flat);
+    if (fieldErr) return res.status(400).json({ error: fieldErr });
+
+    const manifest = registry.getManifest(pluginType);
+    const { config, secrets } = registry.splitConfig(pluginType, flat);
+    const enabled = req.body.enabled !== false; // default true
+    const now = Date.now();
+
+    const instance = await PluginInstance.create({
+      pluginType,
+      category: manifest.category,
+      name,
+      slug,
+      enabled,
+      cron: cron || '0 * * * *',
+      config,
+      created_by: req.user.uid,
+      created_on: now,
+      updated_by: req.user.uid,
+      updated_on: now
+    });
+
+    try {
+      await pluginSecrets.write(instance.id, secrets);
+    } catch (err) {
+      // Most likely the sso-broker policy lacks secret/plugins/* — the
+      // operator needs theta-suite >= v1.30.1. Delete the row so a failed
+      // secret write doesn't strand a half-created instance.
+      await instance.delete().catch(() => {});
+      return res.status(400).json({ error: `Failed to store plugin secrets in OpenBao: ${err.message}. Re-run ./setup.sh with theta-suite >= v1.30.1.` });
+    }
+
+    if (enabled) {
+      await scheduleInstance(instance);
+      await runInstanceNow(instance.id);
+    }
+    res.json({ results: await serialize(instance) });
+  } catch (err) {
+    if (err.name === 'SequelizeUniqueConstraintError') {
+      return res.status(400).json({ error: 'A plugin instance with this slug already exists.' });
+    }
+    next(err);
+  }
+});
+
+// --- Update instance (name/cron/enabled/non-secret config) ---
+router.put('/:id', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    if (!registry.getManifest(inst.pluginType)) return res.status(400).json({ error: `Plugin type ${inst.pluginType} is no longer installed` });
+
+    const updates = {};
+    if (req.body.name !== undefined) updates.name = req.body.name;
+    if (req.body.cron !== undefined) {
+      if (typeof req.body.cron !== 'string' || !req.body.cron.trim()) return res.status(400).json({ error: 'cron must be a non-empty string' });
+      updates.cron = req.body.cron;
+    }
+    if (req.body.enabled !== undefined) updates.enabled = !!req.body.enabled;
+
+    // Non-secret config: split the client's flat config so secret fields are
+    // never written to the DB. Secrets are changed via PUT /:id/secrets.
+    if (req.body.config !== undefined && typeof req.body.config === 'object') {
+      const { config } = registry.splitConfig(inst.pluginType, req.body.config);
+      updates.config = config;
+    }
+
+    updates.updated_by = req.user.uid;
+    updates.updated_on = Date.now();
+
+    const updated = await inst.update(updates);
+
+    // Re-schedule if the schedule-relevant fields moved.
+    if (updates.cron !== undefined || updates.enabled !== undefined) {
+      await scheduleInstance(updated);
+    }
+    res.json({ results: await serialize(updated) });
+  } catch (err) {
+    if (err.name === 'SequelizeUniqueConstraintError') {
+      return res.status(400).json({ error: 'A plugin instance with this slug already exists.' });
+    }
+    next(err);
+  }
+});
+
+// --- Update secrets only ---
+router.put('/:id/secrets', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    if (!registry.getManifest(inst.pluginType)) return res.status(400).json({ error: `Plugin type ${inst.pluginType} is no longer installed` });
+
+    // Keep only declared secret fields; pluginSecrets.write drops blank/MASK
+    // values so an unchanged masked field is a no-op.
+    const { secrets } = registry.splitConfig(inst.pluginType, req.body || {});
+    await pluginSecrets.write(inst.id, secrets);
+    await inst.update({ updated_by: req.user.uid, updated_on: Date.now() });
+    res.json({ results: true });
+  } catch (err) { next(err); }
+});
+
+// --- Test (validate) ---
+router.post('/:id/test', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    const mod = registry.getModule(inst.pluginType);
+    if (typeof mod.validate !== 'function') return res.json({ ok: true, note: 'no validate defined' });
+    const cfg = await pluginSecrets.mergeForRun(inst);
+    const result = await mod.validate(cfg);
+    if (result && result.ok) return res.json(result);
+    return res.status(400).json(result || { ok: false, error: 'validation failed' });
+  } catch (err) {
+    return res.status(400).json({ ok: false, error: err.message });
+  }
+});
+
+// --- Load (enable + schedule + run now) ---
+router.post('/:id/load', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    const updated = await inst.update({ enabled: true, updated_by: req.user.uid, updated_on: Date.now() });
+    await scheduleInstance(updated);
+    await runInstanceNow(updated.id);
+    res.json({ results: await serialize(updated) });
+  } catch (err) { next(err); }
+});
+
+// --- Unload (unschedule + disable) ---
+router.post('/:id/unload', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    await unscheduleInstance(inst.id);
+    const updated = await inst.update({ enabled: false, updated_by: req.user.uid, updated_on: Date.now() });
+    res.json({ results: await serialize(updated) });
+  } catch (err) { next(err); }
+});
+
+// --- Run now (regardless of enabled) ---
+router.post('/:id/run', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    await runInstanceNow(inst.id);
+    res.json({ results: true });
+  } catch (err) { next(err); }
+});
+
+// --- Last-run status ---
+router.get('/:id/runs', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    res.json({ results: { lastRunAt: inst.lastRunAt, lastStatus: inst.lastStatus, lastError: inst.lastError } });
+  } catch (err) { next(err); }
+});
+
+// --- Delete (unschedule + remove secrets + delete row) ---
+router.delete('/:id', async (req, res, next) => {
+  try {
+    const inst = await PluginInstance.get(req.params.id);
+    if (!inst) return res.status(404).json({ error: 'Not found' });
+    await unscheduleInstance(inst.id);
+    await pluginSecrets.remove(inst.id); // best-effort
+    await inst.delete();
+    res.json({ results: true });
+  } catch (err) { next(err); }
+});
+
+module.exports = router;

nodejs/routes/docs.js

@@ -34,7 +34,8 @@ const DOCS = {
 	'oauth-apps':    {title: 'Connecting Apps (SSO)',   file: path.join(__dirname, '../../docs/concepts-oauth-apps.md')},
 	'api-tokens':    {title: 'API Tokens',              file: path.join(__dirname, '../../docs/concepts-api-tokens.md')},
 	directory:       {title: 'Directory & Inventory',   file: path.join(__dirname, '../../docs/directory.md')},
-	agents:          {title: 'Agents & Scheduler',      file: path.join(__dirname, '../../docs/agents.md')},
+	agents:          {title: 'Plugins',                file: path.join(__dirname, '../../docs/plugins.md')},
+	plugins:         {title: 'Plugins',                file: path.join(__dirname, '../../docs/plugins.md')},
 	vault:           {title: 'Vault Secrets',           file: path.join(__dirname, '../../docs/vault.md')},
 
 	overview:        {title: 'Overview',        file: path.join(__dirname, '../../README.md')},

nodejs/routes/index.js

@@ -83,8 +83,14 @@ router.get('/discovery', function(req, res, next) {
   res.redirect('/directory');
 });
 
-router.get('/plugins',  function(req, res, next) {
-  res.redirect('/directory');
+router.get('/plugins', function(req, res, next) {
+  // Plugin instances page — loadable/unloadable, configurable plugin copies
+  // with per-instance secrets in OpenBao. Renders the shell for anyone; the
+  // client gates with app.auth.forceLogin(['app_sso_admin',
+  // 'app_sso_directory_admin','admin']) and the /api/plugins endpoints enforce
+  // the same server-side. Same header-vs-navigation auth model as /conf and
+  // /vault (auth-token is a client-set header, not a cookie).
+  res.render('plugins', {...values});
 });
 
 router.get('/vault', function(req, res) {

nodejs/routes/plugins.js

@@ -1,59 +0,0 @@
-const router = require('express').Router();
-const conf = require('@simpleworkjs/conf');
-const permission = require('../utils/permission');
-
-router.use(async (req, res, next) => {
-  try {
-    await permission.byGroup(req.user, ['app_sso_directory_admin', 'app_sso_admin']);
-    next();
-  } catch(err) {
-    next(err);
-  }
-});
-
-const Redis = require('ioredis');
-const connection = new Redis(process.env.REDIS_URL || 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null });
-const { initScheduler } = require('../services/scheduler');
-
-router.get('/', async (req, res) => {
-  const plugins = conf.discovery && conf.discovery.plugins ? conf.discovery.plugins : {};
-  let overrides = {};
-  try {
-    const data = await connection.hgetall('discovery_plugins');
-    for (const [k, v] of Object.entries(data)) {
-      overrides[k] = JSON.parse(v);
-    }
-  } catch(e) {}
-  
-  // Mask secrets before sending
-  const masked = JSON.parse(JSON.stringify(plugins));
-  for (const name in masked) {
-    masked[name] = { ...masked[name], ...(overrides[name] || {}) };
-    if (masked[name].tokenSecret) masked[name].tokenSecret = '********';
-    if (masked[name].password) masked[name].password = '********';
-  }
-  res.json({ results: masked });
-});
-
-router.put('/:name', async (req, res) => {
-  const name = req.params.name;
-  const updates = req.body;
-  
-  let current = {};
-  try {
-    const data = await connection.hget('discovery_plugins', name);
-    if (data) current = JSON.parse(data);
-  } catch(e) {}
-  
-  if (updates.cron !== undefined) current.cron = updates.cron;
-  if (updates.enabled !== undefined) current.enabled = updates.enabled === true || updates.enabled === 'true';
-  
-  await connection.hset('discovery_plugins', name, JSON.stringify(current));
-  
-  // Re-init scheduler to apply changes
-  await initScheduler(conf.discovery).catch(console.error);
-  
-  res.json({ success: true, message: 'Plugin updated' });
-});
-
-module.exports = router;