From da60310834a9be08b9b5f1dc43c875af62b5643c Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 10 Aug 2026 23:02:01 -0400 Subject: [PATCH] feat(multi-site): auto-configure OpenLDAP replication on join/every run Removes LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS as vars an operator has to hand-set and keep in sync across every site. bootstrap/ site-ldap-register.js (new) asks sso-manager-node's new GET /api/site/ldap-peers (spoke) or GET /directory-admin/ ldap-replication-config (master) for this node's assigned ServerID + current peer list, persists it to /config/ldap-replication.env, and restarts sso-manager only when the computed config actually changed (OpenLDAP's static slapd.conf is only read at process start). Runs on every setup.sh invocation -- both master (peer list grows as spokes join) and spoke. CFG_LDAP_MMR_MANUAL=true skips the automatic step entirely, for a topology outside this theta-suite cluster the script can't derive on its own -- without this escape hatch, an operator's hand-set LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS would get silently overwritten on the next run, since every fresh install starts as a master (the automatic step always runs by default). Bumps sso-manager-node to pick up the new endpoints + SiteSpoke.ldapServerId. --- bootstrap/site-ldap-register.js | 139 ++++++++++++++++++++++++++++++++ docs/MULTI_SITE_SPEC.md | 1 + setup.env.example | 20 +++-- setup.sh | 43 ++++++++++ sso-manager-node | 2 +- 5 files changed, 197 insertions(+), 8 deletions(-) create mode 100644 bootstrap/site-ldap-register.js diff --git a/bootstrap/site-ldap-register.js b/bootstrap/site-ldap-register.js new file mode 100644 index 0000000..ceac5c2 --- /dev/null +++ b/bootstrap/site-ldap-register.js @@ -0,0 +1,139 @@ +#!/usr/bin/env node +/* + * theta-suite site-ldap-register — runs inside the sso-manager container on + * every setup.sh run (both master and spoke) to keep OpenLDAP N-way + * multi-master replication config (docs/replication.md) in sync without an + * operator hand-maintaining LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS. + * + * The master assigns each spoke a unique LDAP_SERVER_ID at join time (same + * mechanism as jump-host's WireGuard mesh index) and derives every site's + * LDAP URL from its already-known HTTPS endpoint -- see sso-manager-node's + * GET /api/site/ldap-peers (spoke-facing) and + * GET /directory-admin/ldap-replication-config (master-local). + * + * This script fetches whichever of those two applies to this node's role, + * and writes the result to /config/ldap-replication.env (KEY=VALUE, the + * same shape setup.env/spoke.env use) if it changed since last run. setup.sh + * sources that file before starting sso-manager on every invocation, and + * restarts the container when this script reports a change -- OpenLDAP's + * static slapd.conf is only read at process start, so a config change needs + * a restart to take effect; there's no live push, which is why this has to + * be re-run periodically (every setup.sh invocation) rather than working + * once at join time and never again, especially on the MASTER, whose peer + * list changes every time a new spoke joins. + * + * docker compose exec sso-manager node /bootstrap/site-ldap-register.js + * + * Self-contained (Node built-ins + global fetch), same rule as + * bootstrap.js/site-join.js -- does NOT require the SSO's internal models. + * + * Output (stdout, KEY=VALUE for setup.sh): LDAP_CONFIG_CHANGED=, + * LDAP_SERVER_ID=, LDAP_REPLICATION_HOSTS=. + * Progress logs go to stderr. + */ +'use strict'; + +const fs = require('fs'); + +const SITE_CONFIG = '/config/site.json'; +const LDAP_CONFIG_FILE = '/config/ldap-replication.env'; +const SSO_INTERNAL = 'http://localhost:3001'; + +const selfUrl = process.argv[2]; + +function log(msg) { console.error('[site-ldap-register] ' + msg); } + +function readPersisted() { + if (!fs.existsSync(LDAP_CONFIG_FILE)) return { LDAP_SERVER_ID: '', LDAP_REPLICATION_HOSTS: '' }; + const out = { LDAP_SERVER_ID: '', LDAP_REPLICATION_HOSTS: '' }; + for (const line of fs.readFileSync(LDAP_CONFIG_FILE, 'utf8').split('\n')) { + const m = line.match(/^([A-Z_]+)=(.*)$/); + if (m && m[1] in out) out[m[1]] = m[2]; + } + return out; +} + +async function fetchMasterConfig() { + const sso = require('/config/sso-secrets.js'); + const adminUid = (sso.bootstrap && sso.bootstrap.adminUid) || 'admin'; + const adminPass = (sso.bootstrap && sso.bootstrap.adminPass) || ''; + + const loginRes = await fetch(`${SSO_INTERNAL}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ uid: adminUid, password: adminPass }), + }); + if (!loginRes.ok) throw new Error(`local admin login failed (${loginRes.status}): ${await loginRes.text().catch(() => '')}`); + const { token } = await loginRes.json(); + if (!token) throw new Error('local admin login returned no token'); + + const cfgRes = await fetch(`${SSO_INTERNAL}/api/directory-admin/ldap-replication-config`, { + headers: { 'auth-token': token }, + }); + if (!cfgRes.ok) throw new Error(`ldap-replication-config failed (${cfgRes.status}): ${await cfgRes.text().catch(() => '')}`); + return cfgRes.json(); +} + +async function fetchSpokeConfig(site, selfUrl) { + const url = `${site.masterUrl.replace(/\/+$/, '')}/api/site/ldap-peers?endpoint=${encodeURIComponent(selfUrl)}`; + const res = await fetch(url, { headers: { Authorization: 'Bearer ' + site.masterJoinKey } }); + const text = await res.text().catch(() => ''); + let data = null; + try { data = JSON.parse(text); } catch (e) { /* not JSON */ } + if (!res.ok) { + if (res.status === 404) { + log('This site is not registered as a spoke on the master yet (join with selfUrl, or re-run site-relay-register.js). Skipping.'); + return null; + } + throw new Error(`ldap-peers failed (${res.status}): ${(data && data.message) || text}`); + } + return data; +} + +async function main() { + if (!fs.existsSync(SITE_CONFIG)) { + log('No /config/site.json yet. Skipping.'); + console.log('LDAP_CONFIG_CHANGED=no'); + return; + } + const site = JSON.parse(fs.readFileSync(SITE_CONFIG, 'utf8')); + + let result; + if (site.isMaster) { + result = await fetchMasterConfig(); + } else { + if (!site.masterUrl || !site.masterJoinKey) { + log('Spoke role but missing masterUrl/masterJoinKey. Skipping.'); + console.log('LDAP_CONFIG_CHANGED=no'); + return; + } + if (!selfUrl) throw new Error('usage: node /bootstrap/site-ldap-register.js (required for a spoke)'); + result = await fetchSpokeConfig(site, selfUrl); + if (!result) { + console.log('LDAP_CONFIG_CHANGED=no'); + return; + } + } + + const serverId = String(result.ldapServerId || ''); + const hosts = (result.peers || []).map((p) => p.ldapHost).filter(Boolean).join(' '); + + const before = readPersisted(); + const changed = before.LDAP_SERVER_ID !== serverId || before.LDAP_REPLICATION_HOSTS !== hosts; + + if (changed) { + fs.writeFileSync(LDAP_CONFIG_FILE, `LDAP_SERVER_ID=${serverId}\nLDAP_REPLICATION_HOSTS=${hosts}\n`); + log(`Replication config changed -- ServerID ${serverId}, ${(result.peers || []).length} peer(s). Wrote ${LDAP_CONFIG_FILE}.`); + } else { + log(`Replication config unchanged -- ServerID ${serverId}, ${(result.peers || []).length} peer(s).`); + } + + console.log(`LDAP_CONFIG_CHANGED=${changed ? 'yes' : 'no'}`); + console.log(`LDAP_SERVER_ID=${serverId}`); + console.log(`LDAP_REPLICATION_HOSTS=${hosts}`); +} + +main().catch((e) => { + console.error('[site-ldap-register] FAILED: ' + e.message); + process.exit(1); +}); diff --git a/docs/MULTI_SITE_SPEC.md b/docs/MULTI_SITE_SPEC.md index 6b3f940..8b151bd 100644 --- a/docs/MULTI_SITE_SPEC.md +++ b/docs/MULTI_SITE_SPEC.md @@ -259,6 +259,7 @@ See [`AGENT_LOCAL_DISCOVERY_SPEC.md`](./AGENT_LOCAL_DISCOVERY_SPEC.md) — split | `setup.env` / `setup.sh` join wiring | **Shipped** — `CFG_MASTER_DIRECTORY_URL` / `CFG_MASTER_DIRECTORY_JOIN_KEY`, `bootstrap/site-join.js` (theta-suite v2.2.0). Also readable from a dedicated `spoke.env` (`spoke.env.example`, layered on top of `setup.env`) for operators who want join-a-cluster config kept separate from the rest of first-run setup. | | Continuous/live replication (vs. one-time export-on-join) | **Shipped** (`sso-manager-node`) — a spoke registers its own endpoint at join time (`POST /api/site/spokes`), and every successful master catalog write fires a fire-and-forget push (`utils/site_replicate.js`) at every registered spoke, which re-pulls a fresh export. Verified end-to-end in `docker-compose.multisite-e2e.yml`. | | Identical-directory signing key | **Shipped** — `POST /api/site/export` includes the master's agent-signing key; a spoke adopts it via `agent_keys.adopt()` on join and every resync. OpenBao secret replication *beyond* this one key is still not built. | +| OpenLDAP N-way multi-master replication auto-config | **Shipped** — the master auto-assigns each spoke a unique `LDAP_SERVER_ID` at registration (`SiteSpoke.ldapServerId`, same pattern as jump-host's mesh index) and derives every site's LDAP URL from its already-known HTTPS endpoint; `theta-suite`'s `bootstrap/site-ldap-register.js` applies it, re-checked on every `setup.sh` run since the peer list grows as spokes join. Verified against real running containers. Known gap: the master's own config only updates when ITS `setup.sh` is re-run, not live the moment a new spoke joins (see `docs/replication.md`). | | Coordinated master promotion (demote the old master as one action) | **Shipped** — `POST /api/site/demote` + `site-promote`'s handoff logic. Fixed two real pre-existing bugs while wiring this in: `site-promote`'s god_admin check read a `req.user.groups` field nothing ever populated (permanently 403'd for everyone), and the read-only write-gate 403'd `site-promote` itself before the handler could run. | | WireGuard gateway-to-gateway mesh (`theta-gateway`) | **Shipped** — `POST /api/mesh/register`/`/join` (join-token bootstrap), `utils/wg_iface.js` (kernel WireGuard, falls back to userspace `wireguard-go`). Verified with a real two-container test: actual encrypted tunnel, real ICMP traffic across it, 0% loss. `wg_iface.removePeer()` also cleans up the kernel routes `setPeer()` added (verified live: routes present after `setPeer`, gone after `removePeer`, own local route untouched), and `DELETE /api/mesh/gateways/:id` exposes it from the mesh UI. | | Cross-component routing (replication over the mesh) | **Shipped** — `utils/site_replicate.js` tries a registered spoke's `meshIp` first (falling back to its public `endpoint` on failure) when pushing resync pings; a spoke with no `meshIp` on file behaves exactly as before. | diff --git a/setup.env.example b/setup.env.example index 0532191..b0cbb18 100644 --- a/setup.env.example +++ b/setup.env.example @@ -152,14 +152,20 @@ CFG_DOMAIN=example.com #CFG_THETA_AGENT_FULL_CONTROL=1 # ── Geo-Location Scaling (N-Way Multi-Master LDAP) ─────────────────────────── -# If deploying this stack across multiple physical sites to provide local HA -# for directory services, you can enable N-Way Multi-Master OpenLDAP replication. -# This requires assigning a unique ID to each site and listing the LDAPS URLs -# of all OTHER sites in the cluster. +# If you're joining a directory cluster (CFG_MASTER_DIRECTORY_URL/spoke.env +# above), N-Way Multi-Master OpenLDAP replication is configured for you +# automatically -- setup.sh's bootstrap/site-ldap-register.js asks the master +# for a unique LDAP_SERVER_ID and the current list of every other site's LDAP +# URL on every run (see docs/replication.md), restarting sso-manager only +# when that config actually changed. Nothing to set here for the common case. # -# Each site MUST have a unique LDAP_SERVER_ID (e.g. 1, 2, 3). -# LDAP_REPLICATION_HOSTS is a space-separated list of the other sites' LDAP URLs. -# Example for Site 1: +# Have a manually-coordinated LDAP MMR topology this script can't derive on +# its own (e.g. peers outside this theta-suite cluster)? Set +# CFG_LDAP_MMR_MANUAL=true to skip the automatic step entirely and set +# LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS directly -- without this, the +# automatic step runs on every deployment (every fresh install starts as a +# master) and will overwrite them. +#CFG_LDAP_MMR_MANUAL=true #LDAP_SERVER_ID=1 #LDAP_REPLICATION_HOSTS="ldaps://sso.site2.com:636 ldaps://sso.site3.com:636" # ── Proxy HTTP/HTTPS Defaults ──────────────────────────────────────────────── diff --git a/setup.sh b/setup.sh index 8e3c5f5..26f8ece 100755 --- a/setup.sh +++ b/setup.sh @@ -1060,6 +1060,15 @@ info "Starting bao-renewer (service-token renewal sidecar)..." SSO_GIT_COMMIT="$(git -C sso-manager-node rev-parse --short HEAD 2>/dev/null || echo unknown)" export SSO_GIT_COMMIT env_upsert SSO_GIT_COMMIT "$SSO_GIT_COMMIT" +# OpenLDAP multi-master replication (docs/replication.md, auto-configured -- +# see step 7e below and bootstrap/site-ldap-register.js): pick up whatever +# LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS a PRIOR run already computed, so a +# restart doesn't silently drop back to standalone (no LDAP_SERVER_ID env at +# all). A truly fresh install has no file yet -- that's fine, it just starts +# standalone until step 7e computes and applies real values. Skipped under +# CFG_LDAP_MMR_MANUAL=true so a manually hand-set LDAP_SERVER_ID/ +# LDAP_REPLICATION_HOSTS in setup.env isn't clobbered by a stale auto file. +[[ "${CFG_LDAP_MMR_MANUAL:-false}" != "true" && -f "$CONFIG_DIR/ldap-replication.env" ]] && parse_kv_file "$CONFIG_DIR/ldap-replication.env" info "Building + starting sso-manager (first run builds the image; this takes a while)..." "${COMPOSE[@]}" up -d --build sso-manager @@ -1322,6 +1331,40 @@ if [[ "${CFG_SPOKE_NO_INBOUND:-false}" == "true" ]]; then fi fi +# ── 7e. OpenLDAP multi-master replication auto-config (every run) ──────────── +# docs/replication.md: the master assigns each spoke a unique LDAP_SERVER_ID +# and derives every site's LDAP URL automatically (bootstrap/ +# site-ldap-register.js) instead of an operator hand-maintaining +# LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS. Runs on every invocation -- both +# master (its peer list grows as spokes join) and spoke -- and restarts +# sso-manager only when the computed config actually changed, since +# OpenLDAP's static slapd.conf is only read at process start. +# +# CFG_LDAP_MMR_MANUAL=true skips this entirely -- every fresh install starts +# as a master, so without this escape hatch an operator's own hand-set +# LDAP_SERVER_ID/LDAP_REPLICATION_HOSTS (a topology outside this theta-suite +# cluster this script can't derive) would get silently overwritten. +if [[ "${CFG_LDAP_MMR_MANUAL:-false}" == "true" ]]; then + info "CFG_LDAP_MMR_MANUAL=true — skipping automatic LDAP replication config." + LDAP_REG_OUT="" +else +LDAP_REG_OUT=$("${COMPOSE[@]}" exec -T sso-manager node /bootstrap/site-ldap-register.js "https://$CFG_SSO_HOST" 2>&1) || warn "LDAP replication config check failed — check: ${COMPOSE[*]} exec sso-manager node /bootstrap/site-ldap-register.js https://$CFG_SSO_HOST" +echo "$LDAP_REG_OUT" | sed 's/^/[setup] /' +fi +if echo "$LDAP_REG_OUT" | grep -q '^LDAP_CONFIG_CHANGED=yes'; then + info "LDAP replication config changed — restarting sso-manager to apply it..." + [[ -f "$CONFIG_DIR/ldap-replication.env" ]] && parse_kv_file "$CONFIG_DIR/ldap-replication.env" + "${COMPOSE[@]}" up -d --force-recreate sso-manager + info "Waiting for sso-manager to be healthy again..." + for i in $(seq 1 60); do + if docker exec sso-manager wget -q -O- http://localhost:3001/health >/dev/null 2>&1; then + info "sso-manager is healthy."; break + fi + if (( i == 60 )); then warn "sso-manager did not become healthy in 120s after the LDAP config restart. Check: ${COMPOSE[*]} logs sso-manager"; break; fi + sleep 2 + done +fi + # ── 7c. Install theta-agent on the host ────────────────────────────────────── # Controlled by CFG_THETA_AGENT_ENABLE (default: 1 = enabled) CFG_THETA_AGENT_ENABLE="${CFG_THETA_AGENT_ENABLE:-1}" diff --git a/sso-manager-node b/sso-manager-node index 39db290..dfb819a 160000 --- a/sso-manager-node +++ b/sso-manager-node @@ -1 +1 @@ -Subproject commit 39db29026575396c9f34cfb7a426b48c28d9300a +Subproject commit dfb819a71565010ae95721e407fdc247d83d7bff