From d486fb946b46deaa2346e4e3aaa99f19f03955ca Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 10 Aug 2026 22:51:04 -0400 Subject: [PATCH] feat(multi-site): auto-assign LDAP ServerID + replication hosts at join time OpenLDAP N-way multi-master replication (docs/replication.md) required an operator to hand-set LDAP_SERVER_ID (unique per site) and LDAP_REPLICATION_HOSTS (every OTHER site's LDAP URL, kept in sync by hand across every node) -- real coordination work, and easy to get wrong or let drift as sites are added. Automates the coordination the master is already in a position to do: - SiteSpoke gets ldapServerId, auto-assigned (next free from 2 upward, 1 reserved for the master) at registration and reused across re-registrations -- same pattern as jump-host's mesh index. - ldapHost is derived from each site's already-known HTTP(S) endpoint (same hostname, port 636) rather than a separately-configured field that could drift from it. - New utils/ldap_replication.js (nextFreeLdapServerId, ldapHostFor), shared between the spoke-facing GET /api/site/ldap-peers (Bearer site join key, returns this caller's own ID + every peer) and the master-local GET /directory-admin/ldap-replication-config (computes its own config directly from SiteSpoke, no HTTP round-trip needed). Verified against real running containers (docker-compose.multisite-e2e.yml): after a real join, the master's computed config correctly includes the spoke as a peer with an assigned ID, and the spoke's own fetched config matches that ID and correctly excludes itself from its own peer list. Known limitation, documented in docs/replication.md: the master's own LDAP_REPLICATION_HOSTS only gets recomputed when ITS setup.sh is re-run (or an admin re-applies it directly) -- there's no live push to an already-running master when a new spoke joins. A spoke's own config is re-checked on every setup.sh run, which is the common/recurring event; the master side is a documented manual step for now rather than a live hot-reload (which would need OpenLDAP's dynamic cn=config backend -- a bigger change, deliberately out of scope here to avoid risking a live directory's LDAP replication on undertested config). --- docs/replication.md | 35 ++++++++++++++++----- nodejs/models/site_spoke.js | 10 +++++- nodejs/package.json | 2 +- nodejs/routes/api_directory_admin.js | 25 +++++++++++++++ nodejs/routes/api_site.js | 44 +++++++++++++++++++++++++++ nodejs/tests/ldap_replication.test.js | 44 +++++++++++++++++++++++++++ nodejs/utils/ldap_replication.js | 44 +++++++++++++++++++++++++++ test/multisite_join_e2e.js | 24 +++++++++++++++ 8 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 nodejs/tests/ldap_replication.test.js create mode 100644 nodejs/utils/ldap_replication.js diff --git a/docs/replication.md b/docs/replication.md index 06955ad..aed4617 100644 --- a/docs/replication.md +++ b/docs/replication.md @@ -23,32 +23,51 @@ In an N-Way Multi-Master setup, every site runs a fully active OpenLDAP server ( ## Configuration -To enable replication, you must pass two environment variables to the `sso-manager` container: +The container's entrypoint reads two environment variables to configure this +-- `LDAP_SERVER_ID` (a unique integer for this node) and +`LDAP_REPLICATION_HOSTS` (a space-separated list of every **other** node's +LDAP URL) -- and, when both are set, automatically loads the `syncprov` +module, enables `mirrormode`, and generates the necessary `syncrepl` blocks +in `/etc/openldap/slapd.conf`. -1. `LDAP_SERVER_ID`: A unique integer for this node (e.g., `1`, `2`, `3`). This MUST be unique across the cluster. -2. `LDAP_REPLICATION_HOSTS`: A space-separated list of the LDAP URLs of all **other** nodes in the cluster. +**If you're using `theta-suite`'s `setup.sh`, you don't set these by hand.** +The master assigns each spoke a unique `LDAP_SERVER_ID` at join time (the +same way it assigns a WireGuard mesh index), and `LDAP_REPLICATION_HOSTS` is +derived automatically from every site's already-known HTTPS endpoint +(`ldaps://:636`) -- see `GET /api/site/ldap-peers` (spoke) and +`GET /api/directory-admin/ldap-replication-config` (master), and +`theta-suite`'s `bootstrap/site-ldap-register.js`, which re-checks on every +`setup.sh` run since the peer list changes as new spokes join. -### Example using `theta-env` / Docker Compose +Setting the two env vars directly still works (e.g. a non-`theta-suite` +deployment) -- example using three manually-configured nodes: -**Site 1 (`setup.env` or `docker-compose.yml`)** +**Site 1** ```env LDAP_SERVER_ID=1 LDAP_REPLICATION_HOSTS="ldaps://sso.site2.com:636 ldaps://sso.site3.com:636" ``` -**Site 2 (`setup.env` or `docker-compose.yml`)** +**Site 2** ```env LDAP_SERVER_ID=2 LDAP_REPLICATION_HOSTS="ldaps://sso.site1.com:636 ldaps://sso.site3.com:636" ``` -**Site 3 (`setup.env` or `docker-compose.yml`)** +**Site 3** ```env LDAP_SERVER_ID=3 LDAP_REPLICATION_HOSTS="ldaps://sso.site1.com:636 ldaps://sso.site2.com:636" ``` -Once configured, the container's entrypoint will automatically load the `syncprov` module, enable `mirrormode`, and generate the necessary `syncrepl` blocks in `/etc/openldap/slapd.conf`. +**A known limitation of the automatic path**: the *master's* own +`LDAP_REPLICATION_HOSTS` only gets recomputed when its `setup.sh` is +re-run (or the operator re-applies it directly) -- there's no live push +telling the master's already-running container about a spoke that joined +five minutes ago. A spoke's own config, by contrast, is re-checked and +applied on every `setup.sh` run there, which is the common/recurring event. +Re-run `setup.sh` on the master after bringing up a new spoke to pick up the +new peer and restart replication with it. ## User Locations diff --git a/nodejs/models/site_spoke.js b/nodejs/models/site_spoke.js index 61ddaae..787b083 100644 --- a/nodejs/models/site_spoke.js +++ b/nodejs/models/site_spoke.js @@ -39,7 +39,15 @@ class SiteSpoke extends Model { noInbound: { type: 'boolean', default: false }, meshIp: { type: 'string' }, publicHost: { type: 'string' }, - relayNote: { type: 'string' } + relayNote: { type: 'string' }, + // OpenLDAP multi-master replication (docs/replication.md): a unique + // small integer this spoke's slapd.conf ServerID must use. Assigned + // once at registration (see api_site.js's nextFreeLdapServerId), + // reused on re-registration -- a spoke that re-registers after a + // restart must not get bumped to a new ID, same reasoning as + // jump-host's meshIndex. The master reserves 1 for itself, never + // assigned here. + ldapServerId: { type: 'integer' } }; toPublic() { diff --git a/nodejs/package.json b/nodejs/package.json index 9797763..06ce160 100755 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "scripts": { "start": "node ./bin/www", "dev": "npx nodemon --ignore public/ ./bin/www", - "test": "NODE_ENV=test jest tests/groups.test.js tests/subtypes.test.js tests/site_join.test.js tests/site_config.test.js tests/site_replicate.test.js tests/proxy_client.test.js tests/reconciler.test.js tests/nmap_plugin.test.js tests/jump_client.test.js --forceExit" + "test": "NODE_ENV=test jest tests/groups.test.js tests/subtypes.test.js tests/site_join.test.js tests/site_config.test.js tests/site_replicate.test.js tests/proxy_client.test.js tests/reconciler.test.js tests/nmap_plugin.test.js tests/jump_client.test.js tests/ldap_replication.test.js --forceExit" }, "jest": { "testEnvironment": "node", diff --git a/nodejs/routes/api_directory_admin.js b/nodejs/routes/api_directory_admin.js index eb92911..99beb19 100644 --- a/nodejs/routes/api_directory_admin.js +++ b/nodejs/routes/api_directory_admin.js @@ -962,6 +962,7 @@ const siteConfig = require('../utils/site_config'); const { siteIsFresh } = require('../utils/site_join'); const { Agent } = require('../models/agent'); const { SiteSpoke } = require('../models/site_spoke'); +const { ldapHostFor } = require('../utils/ldap_replication'); // probeMasterHealth checks whether this (spoke) node can reach its master over // the site join key. The master's /api/site/ping is deliberately lightweight. @@ -1028,6 +1029,30 @@ router.get('/site-status', async (req, res, next) => { } catch (err) { next(err); } }); +// OpenLDAP multi-master replication config for THIS node (docs/replication.md). +// Master-only: the master already has every registered spoke's info locally +// (SiteSpoke), so it can compute its own ServerID (always 1) + full peer +// list without an HTTP round-trip. A spoke gets its config from the master +// directly instead (GET /api/site/ldap-peers -- see bootstrap/ +// site-ldap-register.js in theta-suite, which calls whichever of the two +// applies to this node's role). +router.get('/ldap-replication-config', async (req, res, next) => { + try { + const cfg = siteConfig.get(); + if (!cfg.isMaster) { + return res.status(400).json({ status: 'error', message: 'this node is a spoke -- fetch replication config from the master via GET /api/site/ldap-peers instead' }); + } + const spokes = await SiteSpoke.list(); + const peers = []; + for (const s of spokes) { + if (!s.ldapServerId) continue; + const host = ldapHostFor(s.endpoint); + if (host) peers.push({ ldapServerId: s.ldapServerId, ldapHost: host }); + } + res.json({ status: 'ok', ldapServerId: 1, peers }); + } catch (err) { next(err); } +}); + router.post('/site-promote', async (req, res, next) => { try { // god_admin privilege check. This used to read req.user.groups, which diff --git a/nodejs/routes/api_site.js b/nodejs/routes/api_site.js index 63c66bb..113e109 100644 --- a/nodejs/routes/api_site.js +++ b/nodejs/routes/api_site.js @@ -42,6 +42,8 @@ function logAudit(action, details) { console.log(JSON.stringify({ timestamp: new Date().toISOString(), component: 'site', action, ...details })); } +const { nextFreeLdapServerId, ldapHostFor } = require('../utils/ldap_replication'); + // slurpLdif dumps the local LDAP tree with slapcat (the sso-manager container // carries an OpenLDAP build with slapcat on PATH). async function slurpLdif() { @@ -136,6 +138,7 @@ router.post('/spokes', async (req, res, next) => { endpoint, pushToken: SiteSpoke.generatePushToken(), created_on: now, + ldapServerId: await nextFreeLdapServerId(), ...patch }); } @@ -160,6 +163,47 @@ router.post('/spokes', async (req, res, next) => { } catch (e) { next(e); } }); +// ── LDAP replication peer list (SPOKE-callable, Bearer site join key) ─────── +// OpenLDAP multi-master replication (docs/replication.md) needs each site to +// know its own ServerID plus every OTHER site's LDAPS URL. The master +// coordinates ID assignment (nextFreeLdapServerId, above); this is how a +// spoke asks "what's my ID, and who are my peers" -- called by +// theta-suite's bootstrap/site-ldap-register.js on every setup.sh run, not +// just once at join time, since the peer list changes as other spokes join. +// Same join-key auth as /spokes (a spoke already has this stored from its +// own join). `endpoint` identifies the CALLER so it can be excluded from its +// own peer list -- same identity SiteSpoke.list() keys registration on. +router.get('/ldap-peers', async (req, res, next) => { + try { + const auth = req.headers.authorization || ''; + const rawKey = auth.startsWith('Bearer ') ? auth.slice(7).trim() : ''; + const key = await SiteJoinKey.authenticate(rawKey); + if (!key) return res.status(401).json({ status: 'error', message: 'invalid or revoked site join key' }); + + const callerEndpoint = req.query.endpoint; + if (!callerEndpoint) { + return res.status(400).json({ status: 'error', message: 'endpoint query param is required' }); + } + + const cfg = siteConfig.get(); + const masterHost = ldapHostFor(cfg.masterUrl || req.protocol + '://' + req.get('host')); + const spokes = await SiteSpoke.list(); + const caller = spokes.find((s) => s.endpoint === callerEndpoint); + if (!caller || !caller.ldapServerId) { + return res.status(404).json({ status: 'error', message: 'this endpoint is not a registered spoke -- register via POST /api/site/spokes first' }); + } + + const peers = [{ ldapServerId: 1, ldapHost: masterHost }]; + for (const s of spokes) { + if (s.endpoint === callerEndpoint || !s.ldapServerId) continue; + const host = ldapHostFor(s.endpoint); + if (host) peers.push({ ldapServerId: s.ldapServerId, ldapHost: host }); + } + + res.json({ status: 'ok', ldapServerId: caller.ldapServerId, peers }); + } catch (e) { next(e); } +}); + // ── Resync (SPOKE side, Bearer pushToken; no admin session) ───────────────── // The receiving end of utils/site_replicate.js's fire-and-forget push: the // master pings this when its catalog changes. Deliberately just diff --git a/nodejs/tests/ldap_replication.test.js b/nodejs/tests/ldap_replication.test.js new file mode 100644 index 0000000..1082c2b --- /dev/null +++ b/nodejs/tests/ldap_replication.test.js @@ -0,0 +1,44 @@ +require('./setup'); +const { SiteSpoke } = require('../models/site_spoke'); +const { nextFreeLdapServerId, ldapHostFor } = require('../utils/ldap_replication'); + +describe('ldap_replication', () => { + beforeEach(async () => { + const all = await SiteSpoke.list(); + for (const s of all) await s.delete(); + }); + + describe('ldapHostFor', () => { + test('derives ldaps://:636 from an http(s) endpoint, ignoring its own port', () => { + expect(ldapHostFor('https://sso.site2.example.com')).toBe('ldaps://sso.site2.example.com:636'); + expect(ldapHostFor('https://sso.site2.example.com:8443')).toBe('ldaps://sso.site2.example.com:636'); + expect(ldapHostFor('http://sso.site3.example.com')).toBe('ldaps://sso.site3.example.com:636'); + }); + + test('returns null for an unparseable endpoint', () => { + expect(ldapHostFor('not-a-url')).toBeNull(); + expect(ldapHostFor('')).toBeNull(); + }); + }); + + describe('nextFreeLdapServerId', () => { + test('starts at 2 (1 is reserved for the master) when no spokes are registered', async () => { + await expect(nextFreeLdapServerId()).resolves.toBe(2); + }); + + test('picks the lowest free id, not just the next highest', async () => { + const now = Math.floor(Date.now() / 1000); + await SiteSpoke.create({ id: 'a', endpoint: 'https://a.example.com', pushToken: 'tok-a', created_on: now, ldapServerId: 2 }); + await SiteSpoke.create({ id: 'b', endpoint: 'https://b.example.com', pushToken: 'tok-b', created_on: now, ldapServerId: 4 }); + + await expect(nextFreeLdapServerId()).resolves.toBe(3); + }); + + test('ignores spokes with no ldapServerId assigned yet', async () => { + const now = Math.floor(Date.now() / 1000); + await SiteSpoke.create({ id: 'c', endpoint: 'https://c.example.com', pushToken: 'tok-c', created_on: now }); + + await expect(nextFreeLdapServerId()).resolves.toBe(2); + }); + }); +}); diff --git a/nodejs/utils/ldap_replication.js b/nodejs/utils/ldap_replication.js new file mode 100644 index 0000000..c0d1157 --- /dev/null +++ b/nodejs/utils/ldap_replication.js @@ -0,0 +1,44 @@ +'use strict'; + +// OpenLDAP multi-master replication (docs/replication.md) config derivation, +// shared between routes/api_site.js (the spoke-facing side: assigns a +// ServerID at registration, serves GET /api/site/ldap-peers) and +// routes/api_directory_admin.js (the master-local side: GET +// /directory-admin/ldap-replication-config computes the master's own +// replication config from the same SiteSpoke registry, no HTTP round-trip +// needed since it already has the data). + +const { SiteSpoke } = require('../models/site_spoke'); + +// The master reserves ServerID 1 for itself; every spoke gets the lowest +// free ID from 2 upward, assigned once at registration and reused across +// re-registrations (SiteSpoke.ldapServerId is only ever set on first +// create). Small max, matching mesh_gateway.js's mesh index -- nothing in +// the OpenLDAP protocol requires a small ServerID, but this deployment's +// docs/examples always have. +const MAX_LDAP_SERVER_ID = 4094; + +async function nextFreeLdapServerId() { + const spokes = await SiteSpoke.list(); + const used = new Set(spokes.map((s) => s.ldapServerId).filter(Boolean)); + for (let i = 2; i <= MAX_LDAP_SERVER_ID; i++) { + if (!used.has(i)) return i; + } + throw new Error(`LDAP server ID space exhausted (max ${MAX_LDAP_SERVER_ID} spokes)`); +} + +// A site's LDAP replication URL, derived from its already-known HTTP(S) +// endpoint rather than requiring a separately-configured field: same +// hostname, LDAPS port 636 -- exactly the convention docs/replication.md's +// own worked examples already use (ldaps://sso.site2.com:636 alongside +// https://sso.site2.com). No new config an operator has to keep in sync. +function ldapHostFor(endpoint) { + try { + const host = new URL(endpoint).hostname; + return `ldaps://${host}:636`; + } catch (e) { + return null; + } +} + +module.exports = { MAX_LDAP_SERVER_ID, nextFreeLdapServerId, ldapHostFor }; diff --git a/test/multisite_join_e2e.js b/test/multisite_join_e2e.js index c3c0a43..0d31312 100644 --- a/test/multisite_join_e2e.js +++ b/test/multisite_join_e2e.js @@ -205,6 +205,30 @@ async function main() { if (spokeCfg.config.isMaster !== false) fail(`spoke should be isMaster:false after join, got ${JSON.stringify(spokeCfg.config)}`); if (!spokeCfg.config.masterUrl) fail('spoke should have masterUrl set after join'); + step('Verifying the master computed its own LDAP replication config (ServerID 1 + the new spoke as a peer)'); + const { body: masterLdapCfg } = await api(MASTER_URL, '/api/directory-admin/ldap-replication-config', { token: masterToken }); + if (masterLdapCfg.ldapServerId !== 1) fail(`master's own ldapServerId should be 1, got ${JSON.stringify(masterLdapCfg)}`); + const spokePeer = (masterLdapCfg.peers || []).find(p => p.ldapHost === 'ldaps://spoke:636'); + if (!spokePeer || typeof spokePeer.ldapServerId !== 'number') { + fail(`master's peer list should include the spoke at ldaps://spoke:636 with an assigned ldapServerId, got ${JSON.stringify(masterLdapCfg.peers)}`); + } + + step('Verifying the spoke can fetch its own assigned LDAP ServerID + peer list from the master'); + const spokeLdapPeersResp = await fetch(`${MASTER_URL}/api/site/ldap-peers?endpoint=${encodeURIComponent('http://spoke:3001')}`, { + headers: { Authorization: 'Bearer ' + joinKey } + }); + const spokeLdapCfg = await spokeLdapPeersResp.json(); + if (spokeLdapPeersResp.status !== 200) fail(`GET /api/site/ldap-peers failed: ${spokeLdapPeersResp.status} ${JSON.stringify(spokeLdapCfg)}`); + if (spokeLdapCfg.ldapServerId !== spokePeer.ldapServerId) { + fail(`spoke's own reported ldapServerId (${spokeLdapCfg.ldapServerId}) should match what the master's peer list assigned it (${spokePeer.ldapServerId})`); + } + const masterAsPeer = (spokeLdapCfg.peers || []).find(p => p.ldapServerId === 1); + if (!masterAsPeer || masterAsPeer.ldapHost !== 'ldaps://master:636') { + fail(`spoke's peer list should include the master (ServerID 1, ldaps://master:636), got ${JSON.stringify(spokeLdapCfg.peers)}`); + } + const selfInOwnPeerList = (spokeLdapCfg.peers || []).some(p => p.ldapServerId === spokeLdapCfg.ldapServerId); + if (selfInOwnPeerList) fail(`spoke's own peer list should not include itself, got ${JSON.stringify(spokeLdapCfg.peers)}`); + step('Verifying the spoke adopted the master\'s pre-join catalog'); const spokeResources = await api(SPOKE_URL, '/api/directory-admin/resources', { token: spokeToken }); const adopted = (spokeResources.body.results || spokeResources.body.resources || spokeResources.body || []);