From 744c85f4bf8cab6c77dc09420cb4260be81663cd Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 10 Aug 2026 21:09:10 -0400 Subject: [PATCH] feat(multi-site): wire no-inbound relay registration into the real bootstrap flow sso-manager-node/jump-host already had the relay-automation mechanism (noInbound/meshIp/publicHost -> theta-proxy route via proxy_client.js, GET /api/mesh/self on jump-host) but nothing in the actual operator bring-up flow could ever reach it -- setup.sh, bootstrap/site-join.js, and setup.env.example had zero wiring for it. Add bootstrap/site-relay-register.js: reads this spoke's own role from /config/site.json, logs into the local jump-host as its bootstrap admin to discover its mesh IP, and registers it with the master. Mesh peering itself stays a manual step (mint/paste a join token, same pattern as the site join key), so this runs on every setup.sh invocation via CFG_SPOKE_NO_INBOUND/CFG_SPOKE_PUBLIC_HOST and is a no-op ("not meshed yet") until an operator has actually meshed the two jump-hosts. Also updates MULTI_SITE_SPEC.md's status table/TODO and the published mesh.md docs page, which still described this as "designed but not automated" after the API-level work had already shipped. --- bootstrap/site-relay-register.js | 124 +++++++++++++++++++++++++++++++ docs/MULTI_SITE_SPEC.md | 16 ++-- docs/jump-host/mesh.md | 20 +++-- setup.env.example | 14 ++++ setup.sh | 19 +++++ 5 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 bootstrap/site-relay-register.js diff --git a/bootstrap/site-relay-register.js b/bootstrap/site-relay-register.js new file mode 100644 index 0000000..990a01d --- /dev/null +++ b/bootstrap/site-relay-register.js @@ -0,0 +1,124 @@ +#!/usr/bin/env node +/* + * theta-suite site-relay-register — runs inside the sso-manager container + * (same pattern as site-join.js) to finish no-inbound relay automation for a + * spoke with no public IP (MULTI_SITE_SPEC.md §5.2). + * + * site-join.js's initial join can't supply a mesh IP: this site's jump-host + * isn't meshed to the master's yet at that point (mesh peering is a manual, + * out-of-band action on both jump-hosts -- mint a join token on the master's + * jump-host, paste it into this site's jump-host "Join a mesh" UI action -- + * the same reason the site join key itself is minted/pasted by hand rather + * than automated). This script is the follow-up: run it (setup.sh does, on + * every run, when CFG_SPOKE_NO_INBOUND is set) once meshing is done, and it + * discovers this jump-host's mesh IP and registers it with the master so + * theta-proxy there can auto-create the relay route (see sso-manager-node's + * utils/proxy_client.js). Safe to run before meshing completes -- reports + * "not meshed yet" and exits 0 so a re-run later just picks it up. + * + * docker compose exec sso-manager node /bootstrap/site-relay-register.js \ + * https://sso.this-site.example.com sso-branch2.master-domain.example.com + * + * Self-contained (Node built-ins + global fetch), same rule as bootstrap.js + * and site-join.js -- it does NOT require the SSO's internal models. It + * reads this node's own spoke role from /config/site.json (written by + * site-join.js) and logs into the LOCAL jump-host as its bootstrap-minted + * local admin (/config/jump-secrets.js) to call jump-host's own + * GET /api/mesh/self. + * + * Output (stdout, KEY=VALUE for setup.sh): RELAY=. + * Progress logs go to stderr. + */ +'use strict'; + +const fs = require('fs'); + +const SITE_CONFIG = '/config/site.json'; +const JUMP_SECRETS = '/config/jump-secrets.js'; +const JUMP_INTERNAL = 'http://jump-host:3002'; + +const selfUrl = process.argv[2]; +const publicHost = process.argv[3]; + +function log(msg) { console.error('[site-relay-register] ' + msg); } + +async function main() { + if (!selfUrl || !publicHost) { + throw new Error('usage: node /bootstrap/site-relay-register.js '); + } + + if (!fs.existsSync(SITE_CONFIG)) { + log('No /config/site.json yet — this node has not joined a master. Nothing to do.'); + console.log('RELAY=not-a-spoke'); + return; + } + const site = JSON.parse(fs.readFileSync(SITE_CONFIG, 'utf8')); + if (site.isMaster || !site.masterUrl || !site.masterJoinKey) { + log('Not a joined spoke (missing masterUrl/masterJoinKey, or this is a master). Nothing to do.'); + console.log('RELAY=not-a-spoke'); + return; + } + + if (!fs.existsSync(JUMP_SECRETS)) { + log('No /config/jump-secrets.js — jump-host has not been provisioned yet. Skipping.'); + console.log('RELAY=skipped'); + return; + } + const jumpSecrets = require(JUMP_SECRETS); + const jumpAdminUser = (jumpSecrets.auth && jumpSecrets.auth.adminUsers && jumpSecrets.auth.adminUsers[0]) || 'jumpadmin'; + const jumpAdminPass = (jumpSecrets.auth && jumpSecrets.auth.localAdminPass) || ''; + if (!jumpAdminPass) { + log('jump-secrets.js has no local admin password. Skipping.'); + console.log('RELAY=skipped'); + return; + } + + const loginRes = await fetch(`${JUMP_INTERNAL}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ uid: jumpAdminUser, password: jumpAdminPass }), + }); + if (!loginRes.ok) { + throw new Error(`jump-host admin login failed (${loginRes.status}): ${await loginRes.text().catch(() => '')}`); + } + const { token: jumpToken } = await loginRes.json(); + if (!jumpToken) throw new Error('jump-host login returned no token'); + + const selfRes = await fetch(`${JUMP_INTERNAL}/api/mesh/self`, { headers: { 'auth-token': jumpToken } }); + if (!selfRes.ok) { + throw new Error(`jump-host mesh self-lookup failed (${selfRes.status}): ${await selfRes.text().catch(() => '')}`); + } + const selfData = await selfRes.json(); + if (!selfData.meshIp) { + log('jump-host is not meshed yet (no mesh IP assigned). Mesh-join it first (jump-host UI), then re-run setup.sh.'); + console.log('RELAY=not-meshed'); + return; + } + log(`Discovered mesh IP ${selfData.meshIp}. Registering with ${site.masterUrl}...`); + + const regRes = await fetch(`${site.masterUrl.replace(/\/+$/, '')}/api/site/spokes`, { + method: 'POST', + headers: { Authorization: 'Bearer ' + site.masterJoinKey, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + endpoint: selfUrl, + siteSlug: site.siteSlug || '', + noInbound: true, + meshIp: selfData.meshIp, + publicHost, + }), + }); + const text = await regRes.text().catch(() => ''); + let data = null; + try { data = JSON.parse(text); } catch (e) { /* not JSON */ } + if (!regRes.ok) { + throw new Error(`relay registration failed (${regRes.status}): ${(data && data.message) || text}`); + } + + log(`Relay: ${(data.relay && data.relay.note) || 'registered'}`); + console.log('RELAY=registered'); +} + +main().catch((e) => { + console.error('[site-relay-register] FAILED: ' + e.message); + process.exit(1); +}); diff --git a/docs/MULTI_SITE_SPEC.md b/docs/MULTI_SITE_SPEC.md index 23485bf..4033524 100644 --- a/docs/MULTI_SITE_SPEC.md +++ b/docs/MULTI_SITE_SPEC.md @@ -8,7 +8,7 @@ > ## Shipped today > - **Join, live replication, promotion** (`sso-manager-node`): a spoke joins via a one-time export over a site join key (`POST /api/site/join-keys` / `/export` / `/join`), then registers its own endpoint so the master can push live resync pings on every catalog write — no longer a one-time snapshot. Promotion (`POST /api/directory-admin/site-promote`) coordinates a real handoff, demoting the old master as one action. Identical agent-signing keys ride the same export/resync path. Read [`sso-manager-node/docs/site-join.md`](https://github.com/theta42/theta-directory/blob/master/docs/site-join.md) and `directory_spec.md` §11 for the endpoint-level detail. > - **Gateway-to-gateway WireGuard mesh** (`theta-gateway`): real site-to-site tunnels via `POST /api/mesh/register`/`/join`, kernel WireGuard with a userspace `wireguard-go` fallback. Verified with an actual two-container encrypted tunnel passing traffic, not a mock. -> - **Not yet connected to each other**: the mesh is a transport layer that exists on its own; `sso-manager-node`'s HTTPS-based join/replicate calls don't route over it yet. That wiring, plus the no-inbound relay it would enable (mechanism verified, automation not built — see status table), is the next layer. +> - **Cross-component routing + no-inbound relay automation**: `sso-manager-node`'s replication traffic now prefers a spoke's mesh IP over the open internet when one is on file (`utils/site_replicate.js`), and a no-inbound spoke's join (`POST /api/site/join` → `/api/site/spokes`) can carry `noInbound`/`meshIp`/`publicHost`, which drives `utils/proxy_client.js` to auto-create the relay route on the master's `theta-proxy` via its existing self-service API token system (reused, not a new credential type). The one piece that stays a manual, out-of-band step is the mesh peering itself (mint a join token on one jump-host, paste it into the other's "Join a mesh" UI) — `theta-suite`'s `bootstrap/site-relay-register.js` (`CFG_SPOKE_NO_INBOUND`/`CFG_SPOKE_PUBLIC_HOST`) picks up from there on the next `setup.sh` run. > - **mDNS local-discovery (Linux + Windows)**: shipped and verified — `theta-gateway` announces (`services/mdns_announce.js`), `theta-agent` discovers and applies a hosts-file override, cleanly reverts when the announcement disappears. Linux was verified end-to-end over real multicast; Windows shipped in `theta-agent` v2.2.0 (CRLF-aware hosts override, `ipconfig /flushdns`, and a /32 host-route pin so the WireGuard tunnel can't swallow the direct LAN path). macOS still needs real testing — see the TODO note. Design scale: a handful of sites (dozen max, 254 hard ceiling — see §4), a few hundred users/hosts total. This is a deliberate, small, trusted-operator deployment, not a hyperscale/adversarial-tenant one — several decisions below (fire-and-forget replication, identical directories) trade blast-radius for simplicity *because* the scale allows it. Don't generalize these choices past that scale without re-deriving them. @@ -260,19 +260,17 @@ See [`AGENT_LOCAL_DISCOVERY_SPEC.md`](./AGENT_LOCAL_DISCOVERY_SPEC.md) — split | Continuous/live replication (vs. one-time export-on-join) | **Shipped** (`sso-manager-node`) — a spoke registers its own endpoint at join time (`POST /api/site/spokes`), and every successful master catalog write fires a fire-and-forget push (`utils/site_replicate.js`) at every registered spoke, which re-pulls a fresh export. Verified end-to-end in `docker-compose.multisite-e2e.yml`. | | Identical-directory signing key | **Shipped** — `POST /api/site/export` includes the master's agent-signing key; a spoke adopts it via `agent_keys.adopt()` on join and every resync. OpenBao secret replication *beyond* this one key is still not built. | | Coordinated master promotion (demote the old master as one action) | **Shipped** — `POST /api/site/demote` + `site-promote`'s handoff logic. Fixed two real pre-existing bugs while wiring this in: `site-promote`'s god_admin check read a `req.user.groups` field nothing ever populated (permanently 403'd for everyone), and the read-only write-gate 403'd `site-promote` itself before the handler could run. | -| WireGuard gateway-to-gateway mesh (`theta-gateway`) | **Shipped** — `POST /api/mesh/register`/`/join` (join-token bootstrap), `utils/wg_iface.js` (kernel WireGuard, falls back to userspace `wireguard-go`). Verified with a real two-container test: actual encrypted tunnel, real ICMP traffic across it, 0% loss. This is the mesh transport layer only — nothing in `sso-manager-node`'s replication yet routes traffic *over* it; today's site-to-site HTTPS calls (join/export/resync) still go over whatever network path already reaches the target, same as before this layer existed. | -| No-inbound-spoke relay (master proxies a spoke with no public IP) | **Mechanism verified, automation not built.** Confirmed with a standalone test (not `theta-proxy`'s actual Lua/Redis engine, which needs its own dedicated pass to wire safely): a spoke with zero published ports, reachable only via its WG mesh IP, served a request that an external client sent to the master's public port — the master terminated the connection and relayed over the tunnel. So the underlying idea works; what's missing is `theta-proxy` automatically creating that relay route when a no-inbound spoke registers (needs a real service-to-service credential between `sso-manager-node` and `theta-proxy`/`theta-gateway` that doesn't exist yet — a new integration, not a small wiring task), and today's HTTPS-based join/replicate still requires the spoke to reach the master's API directly (and vice versa for export), so a spoke with zero inbound *and* zero outbound path still can't join at all. | +| WireGuard gateway-to-gateway mesh (`theta-gateway`) | **Shipped** — `POST /api/mesh/register`/`/join` (join-token bootstrap), `utils/wg_iface.js` (kernel WireGuard, falls back to userspace `wireguard-go`). Verified with a real two-container test: actual encrypted tunnel, real ICMP traffic across it, 0% loss. `wg_iface.removePeer()` also cleans up the kernel routes `setPeer()` added (verified live: routes present after `setPeer`, gone after `removePeer`, own local route untouched), and `DELETE /api/mesh/gateways/:id` exposes it from the mesh UI. | +| Cross-component routing (replication over the mesh) | **Shipped** — `utils/site_replicate.js` tries a registered spoke's `meshIp` first (falling back to its public `endpoint` on failure) when pushing resync pings; a spoke with no `meshIp` on file behaves exactly as before. | +| No-inbound-spoke relay (master proxies a spoke with no public IP) | **Shipped at the API/automation layer, wired into the real bootstrap flow.** `POST /api/site/join`/`/api/site/spokes` accept `noInbound`/`meshIp`/`publicHost` and call `utils/proxy_client.js`, which mints/reuses a `theta-proxy` self-service API token (`prx_...`, OpenBao `secret/integrations/theta-proxy`) and calls the proxy's real Host API to create or update the relay route — verified against a real running `theta-proxy` container (`GET /api/host/:item`'s actual `{item, results: {...}}` response shape, not the flat shape first assumed). `theta-suite`'s `bootstrap/site-relay-register.js` + `CFG_SPOKE_NO_INBOUND`/`CFG_SPOKE_PUBLIC_HOST` (`setup.env.example`) drive it from the operator-facing bring-up flow, re-run automatically on every `setup.sh` invocation until the jump-host mesh IP is discoverable. What's still a manual step, deliberately: the gateway-to-gateway mesh *peering* itself (mint a join token on one jump-host, paste it into the other's UI) — same pattern as minting/pasting a site join key, not something an unattended script should do blind. A spoke with zero inbound *and* zero outbound path still can't join at all (join/export still need the spoke to reach the master's API directly). | | mDNS local-discovery (Linux) | **Shipped** — `theta-gateway` announces (`services/mdns_announce.js`, opt-in via `THETA_LOCAL_DISCOVERY_HOSTS`), `theta-agent` discovers and applies a hosts-file override (`local_discovery.go`, opt-in via `prefer_local_directory`). Verified end-to-end with real containers over real multicast: announce → discover → apply → clean revert on disappearance, all confirmed. Caught two real bugs along the way (`mdns.Lookup()`'s IPv6 query aborting the whole lookup even after a valid IPv4 response arrived; `rename()` failing with EBUSY over a bind-mounted `/etc/hosts`, common in every container runtime) — see the commit messages in `theta-agent`. | | mDNS local-discovery (Windows) | **Shipped** — `theta-agent` v2.2.0: Windows hosts override (`%SystemRoot%\System32\drivers\etc\hosts`, CRLF-aware, `ipconfig /flushdns` after each change — reachable because the agent runs as a SYSTEM service, so the elevation question resolved in our favor), plus a /32 host-route pin via the owning local interface (`route.exe add ... metric 1`) so the WireGuard mesh tunnel can't swallow the direct LAN path, and a prompt WS reconnect on apply/revert. Tests run the real Windows write path on the Windows CI leg. | | mDNS local-discovery (macOS) | Not built — the hosts override compiles on darwin via the shared unix path, but macOS still needs `dscacheutil -flushcache` and real hardware testing (mDNSResponder behavior, hosts-file vs. native Bonjour — see Appendix B §3). Being built on a real macOS VM. | -### TODO — what's actually left, in dependency order +### TODO — what's actually left -1. **Service-to-service auth model for internal components** (`sso-manager-node` ↔ `theta-proxy`/`theta-gateway`). No unified credential exists today — the site join key (spoke↔master) and the mesh join token (gateway↔gateway) are two separately-invented patterns for the same underlying problem. Items 2 and 3 below both need this; design it once here rather than letting a third integration invent a third pattern. -2. **Cross-component routing** — route `sso-manager-node`'s HTTPS replication traffic (join/export/resync) over the WireGuard mesh instead of the open internet, now that the mesh exists as its own transport layer. Blocked on #1. -3. **No-inbound relay automation** — the relay mechanism itself is verified (master terminates + relays to a spoke with zero inbound path — see status table), but nothing in `theta-proxy` creates that route automatically when a no-inbound spoke registers. Blocked on #1. -4. **Full secret replication** — only the agent-signing key is replicated today. LDAP admin credentials, JWT secrets, and other per-deployment secrets still differ per site, which complicates full disaster recovery. -5. **Mesh peer cleanup** — `wg_iface.removePeer()` doesn't remove the kernel routes `setPeer()` adds (flagged in code; not yet exercised because nothing removes a mesh peer today). +1. **Full secret replication** — only the agent-signing key is replicated today. LDAP admin credentials, JWT secrets, and other per-deployment secrets still differ per site, which complicates full disaster recovery. **Paused pending a real-deployment question independent of the code**: this repo's own `conf/secrets.js` was found to contain committed real credentials during this work (LDAP bind, SMTP, VoIP.ms) — see the git-remediation note elsewhere in this repo's history. Building a feature that copies live secrets to additional sites shouldn't proceed until provider-side rotation of those specific credentials is confirmed done; the mechanism itself (generic secret sync, never touching those particular values) can still be designed without that answer. +2. Service-to-service auth, cross-component routing, no-inbound relay automation, and mesh peer cleanup (the four items formerly listed here) are **done** — see the status table above. What remains genuinely open in that area is documented there inline (mesh peering stays a manual step by design; zero-inbound-and-zero-outbound spokes still can't join). **mDNS local-discovery, macOS** is deliberately not listed above: the Linux and Windows sides are shipped and verified (`theta-agent` v2.2.0), and macOS is being built on a real macOS VM where the darwin-specific behavior (mDNSResponder/DNS-cache) can actually be tested. Check `theta-agent`'s recent history before assuming it's still open. diff --git a/docs/jump-host/mesh.md b/docs/jump-host/mesh.md index 14249ae..824de37 100644 --- a/docs/jump-host/mesh.md +++ b/docs/jump-host/mesh.md @@ -48,13 +48,17 @@ the hard ceiling this addressing scheme supports. - `NET_ADMIN` capability (or equivalent) on the container/host running the gateway, to create the WireGuard interface. -## Not yet connected to directory sync +## Connected to directory sync -This mesh is a networking layer on its own. [Theta Directory's multi-site -join](../sso/multi-site.html) (catalog + LDAP replication between a master -and its spokes) does not currently route its traffic over this mesh — the -two features work independently today. Routing directory sync over the mesh, -and using the mesh to reach a spoke site with no inbound access of its own, -are both designed but not yet automated — see the [architecture +[Theta Directory's multi-site join](../sso/multi-site.html) (catalog + LDAP +replication between a master and its spokes) prefers this mesh once it's up: +a spoke that's registered a mesh IP gets its live resync pushes routed over +the tunnel instead of the open internet, falling back to its public endpoint +if the mesh path fails. A spoke with no public IP at all can also register as +no-inbound (`CFG_SPOKE_NO_INBOUND` in `theta-suite`'s `setup.env`) so the +master auto-creates a relay route through its own `theta-proxy` — the master +terminates TLS for that spoke's hostname and relays over this mesh. The mesh +peering itself (this page) stays a manual step on both sides; directory join +and relay registration pick up from there. See the [architecture spec](https://github.com/theta42/theta-suite/blob/master/docs/MULTI_SITE_SPEC.md) -for current status. +for the full detail. diff --git a/setup.env.example b/setup.env.example index 401f4e8..adfb31c 100644 --- a/setup.env.example +++ b/setup.env.example @@ -64,6 +64,20 @@ CFG_DOMAIN=example.com #CFG_MASTER_DIRECTORY_URL=https://sso.master.example.com #CFG_MASTER_DIRECTORY_JOIN_KEY=stj_9f2e... +# No public IP at all (CGNAT, etc.)? The master can still reach this spoke by +# relaying over the gateway-to-gateway WireGuard mesh instead of the open +# internet (MULTI_SITE_SPEC.md §5.2) -- but the mesh peering itself is a +# manual, out-of-band step on BOTH jump-hosts (mint a mesh join token on the +# master's jump-host, paste it into this site's jump-host "Join a mesh" UI +# action) that can't run unattended inside this script. Once that's done, set +# these two and re-run setup.sh: it discovers this jump-host's assigned mesh +# IP (GET /api/mesh/self) and registers it with the master, which then +# auto-creates the relay route on its own theta-proxy. Safe to leave set +# before meshing -- setup.sh just reports "not meshed yet" and skips until a +# later re-run finds the mesh IP. +#CFG_SPOKE_NO_INBOUND=true +#CFG_SPOKE_PUBLIC_HOST=sso-branch2.master-domain.example.com + # ── Optional outbound HTTP(S) proxy ────────────────────────────────────────── # For an isolated/offline/corporate-network test host that only reaches the # internet through an upstream HTTP proxy — NOT the theta42 "proxy" app. diff --git a/setup.sh b/setup.sh index 786444d..de4690b 100755 --- a/setup.sh +++ b/setup.sh @@ -1279,6 +1279,25 @@ NODEEOF ) echo "$JUMP_HOSTS_OUT" | sed 's/^/[setup] /' +# ── 7b2. No-inbound relay registration (first-run *and* every re-run) ───────── +# CFG_SPOKE_NO_INBOUND: this site has no public IP, so the master relays to it +# over the gateway-to-gateway WireGuard mesh (MULTI_SITE_SPEC.md §5.2). The +# mesh peering itself is a manual, out-of-band step on both jump-hosts (mint a +# join token on the master's jump-host, paste it into this site's jump-host +# "Join a mesh" UI action) -- it can't run unattended here, and it commonly +# happens AFTER this first setup.sh run finishes. So this step runs on every +# invocation, not just first-run: it discovers this jump-host's mesh IP and +# (re-)registers it with the master, and is a no-op until meshing is done. +if [[ "${CFG_SPOKE_NO_INBOUND:-false}" == "true" ]]; then + if [[ -z "${CFG_SPOKE_PUBLIC_HOST:-}" ]]; then + warn "CFG_SPOKE_NO_INBOUND=true but CFG_SPOKE_PUBLIC_HOST is unset — skipping relay registration." + else + info "Checking no-inbound relay registration (CFG_SPOKE_PUBLIC_HOST=${CFG_SPOKE_PUBLIC_HOST})..." + "${COMPOSE[@]}" exec -T sso-manager node /bootstrap/site-relay-register.js \ + "https://$CFG_SSO_HOST" "$CFG_SPOKE_PUBLIC_HOST" || warn "relay registration did not complete — check: ${COMPOSE[*]} exec sso-manager node /bootstrap/site-relay-register.js https://$CFG_SSO_HOST $CFG_SPOKE_PUBLIC_HOST" + fi +fi + # ── 7c. Install theta-agent on the host ────────────────────────────────────── # Controlled by CFG_THETA_AGENT_ENABLE (default: 1 = enabled) CFG_THETA_AGENT_ENABLE="${CFG_THETA_AGENT_ENABLE:-1}"