feat(mesh): real gateway-to-gateway WireGuard mesh (site-to-site tunnels)

The existing WireGuard code (models/wg_site.js, routes/wireguard.js) is the
roaming-client/exit-node feature -- individual peer configs an admin hands
out, not gateway-to-gateway mesh peering. This adds the latter, per
MULTI_SITE_SPEC.md §4: two theta-gateway instances mesh by one calling the
other's POST /api/mesh/register with a join token (minted via
POST /api/mesh/join-tokens, admin-gated); both sides end up with a live
wg0 peer for the other, mesh-indexed per Appendix A's addressing
(172.24.<idx>.0/16 + 10.<idx>.0.0/16, idx 1-254).

- utils/wg_iface.js: brings up the local interface, preferring in-kernel
  WireGuard (ip link add type wireguard) and falling back to userspace
  wireguard-go when the kernel module isn't available. Both packages
  added to the Dockerfile.
- utils/mesh_addressing.js: pure addressing math, unit tested
  (test/unit/mesh_addressing.test.js).
- models/mesh_gateway.js: Redis-backed registry of known peer gateways
  (same pattern as wg_site.js), assigns + persists mesh indexes.
- utils/mesh_join_token.js: single-use bootstrap credential, same
  GETDEL-on-Redis pattern already used on the theta-directory side.
- routes/mesh.js: /join-tokens (admin), /register (bearer token, no
  session -- called by a remote gateway), /join (admin, initiates from
  this side), /gateways (admin, list).

Verified with a REAL two-container test (not mocked): two independent
containers, each running this actual code, meshed via a live join-token
handshake, brought up real kernel WireGuard interfaces, and passed ICMP
traffic across the resulting encrypted tunnel end to end (0% packet
loss). That test caught a real bug worth calling out: `wg set ... peer
... allowed-ips` only configures WireGuard's own crypto-routing table --
it does NOT add a kernel route for that destination (wg-quick normally
does this as a separate step; we don't use wg-quick). A real encrypted
handshake completed between the two containers with the route missing,
and ping still showed 100% loss until setPeer() was fixed to add the
corresponding `ip route add <allowed-ip> dev <iface>` itself.
This commit is contained in:
2026-08-10 17:23:19 -04:00
parent 02767cac47
commit 99276f4ee5
9 changed files with 495 additions and 1 deletions
+1
View File
@@ -17,6 +17,7 @@ FROM node:22-bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
redis-server dumb-init ca-certificates \ redis-server dumb-init ca-certificates \
iproute2 wireguard-tools wireguard-go \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
WORKDIR /app WORKDIR /app
+85
View File
@@ -0,0 +1,85 @@
'use strict';
// Registry of peer theta-gateway instances this gateway has meshed with —
// raw Redis, same pattern as wg_site.js/audit_event.js. Each registration
// carries what's needed to configure a local WireGuard peer entry for them:
// public key, reachable endpoint, and the mesh IP this gateway assigned them
// (MULTI_SITE_SPEC.md's one-octet-per-site addressing, 172.24.<idx>.0/16 +
// 10.<idx>.0.0/16, idx 1-254).
//
// Redis keys:
// mesh_gateway:<id> — hash of gateway fields
// mesh_gateway_index — sorted set (score = createdAt, value = id)
const crypto = require('crypto');
const conf = require('@simpleworkjs/conf');
const { getRedis } = require('./index');
const MAX_MESH_INDEX = 254;
const P = () => conf.redis.prefix;
const idxKey = () => `${P()}mesh_gateway_index`;
const gatewayKey = (id) => `${P()}mesh_gateway:${id}`;
function serialize(obj) {
const out = {};
for (const [k, v] of Object.entries(obj)) {
out[k] = String(v == null ? '' : v);
}
return out;
}
function deserialize(h) {
if (!h || !h.id) return null;
return { ...h, meshIndex: Number(h.meshIndex || 0), createdAt: Number(h.createdAt || 0), lastSeenAt: Number(h.lastSeenAt || 0) };
}
async function list() {
const redis = await getRedis();
const ids = await redis.zRange(idxKey(), 0, -1);
const out = [];
for (const id of ids) {
const g = deserialize(await redis.hGetAll(gatewayKey(id)));
if (g) out.push(g);
}
return out;
}
async function findByPublicKey(publicKey) {
const all = await list();
return all.find((g) => g.publicKey === publicKey) || null;
}
function nextFreeMeshIndex(existing) {
const used = new Set(existing.map((g) => g.meshIndex).filter(Boolean));
for (let i = 1; i <= MAX_MESH_INDEX; i++) {
if (!used.has(i)) return i;
}
throw new Error(`Mesh index space exhausted (max ${MAX_MESH_INDEX} gateways)`);
}
// Register (or re-register, idempotent by publicKey) a peer gateway.
// Re-registering the same public key updates its endpoint/siteSlug but
// reuses its existing mesh index -- a gateway that re-registers after a
// restart must not get bumped to a new mesh subnet.
async function register({ publicKey, endpoint, siteSlug }) {
const redis = await getRedis();
const existing = await list();
const already = existing.find((g) => g.publicKey === publicKey);
const now = Date.now();
if (already) {
const updated = { ...already, endpoint, siteSlug: siteSlug || already.siteSlug, lastSeenAt: now };
await redis.hSet(gatewayKey(already.id), serialize(updated));
return updated;
}
const id = crypto.randomBytes(8).toString('hex');
const meshIndex = nextFreeMeshIndex(existing);
const gateway = { id, publicKey, endpoint, siteSlug: siteSlug || '', meshIndex, createdAt: now, lastSeenAt: now };
await redis.hSet(gatewayKey(id), serialize(gateway));
await redis.zAdd(idxKey(), { score: now, value: id });
return gateway;
}
module.exports = { list, findByPublicKey, register, MAX_MESH_INDEX };
+1 -1
View File
@@ -14,7 +14,7 @@
"scripts": { "scripts": {
"start": "node ./bin/www", "start": "node ./bin/www",
"dev": "npx nodemon --ignore public/ ./bin/www", "dev": "npx nodemon --ignore public/ ./bin/www",
"test": "NODE_ENV=test node --test --test-force-exit test/unit/access.test.js test/unit/host_keys.test.js test/unit/no_native_dialogs.test.js test/unit/target_match.test.js test/unit/username_grammar.test.js test/unit/wireguard.test.js", "test": "NODE_ENV=test node --test --test-force-exit test/unit/access.test.js test/unit/host_keys.test.js test/unit/no_native_dialogs.test.js test/unit/target_match.test.js test/unit/username_grammar.test.js test/unit/wireguard.test.js test/unit/mesh_addressing.test.js",
"test:unit": "NODE_ENV=test node --test --test-force-exit test/unit/*.test.js", "test:unit": "NODE_ENV=test node --test --test-force-exit test/unit/*.test.js",
"test:integration": "NODE_ENV=test node --test --test-force-exit test/integration/*.test.js" "test:integration": "NODE_ENV=test node --test --test-force-exit test/integration/*.test.js"
}, },
+5
View File
@@ -19,4 +19,9 @@ router.use('/', middleware.auth, middleware.requireJumpAdmin, require('./jump'))
// WireGuard peer + site management — admin only. // WireGuard peer + site management — admin only.
router.use('/wireguard', middleware.auth, middleware.requireJumpAdmin, require('./wireguard')); router.use('/wireguard', middleware.auth, middleware.requireJumpAdmin, require('./wireguard'));
// Gateway-to-gateway mesh — mixed auth (register/register-* are called by a
// remote gateway with a bearer join token, not an admin session; join-tokens
// mint + join are admin-gated). See routes/mesh.js for the per-route gates.
router.use('/mesh', require('./mesh'));
module.exports = router; module.exports = router;
+164
View File
@@ -0,0 +1,164 @@
'use strict';
// Gateway-to-gateway WireGuard mesh — real site-to-site tunnels, not the
// roaming-client/exit-node feature in routes/wireguard.js. Two gateways mesh
// by one calling the other's POST /api/mesh/register with a join token; both
// sides end up with a live wg0 peer entry for the other, addressed per
// MULTI_SITE_SPEC.md's one-octet mesh index (172.24.<idx>.0/16,
// 10.<idx>.0.0/16, idx 1-254, assigned by whichever gateway is registering
// the caller).
//
// Local interface name is fixed at THETA_MESH_IFACE (default wg-mesh) —
// deliberately separate from the roaming-client interface so the two
// features never fight over the same wg0.
const express = require('express');
const middleware = require('../middleware/auth');
const conf = require('@simpleworkjs/conf');
const meshGateway = require('../models/mesh_gateway');
const meshJoinToken = require('../utils/mesh_join_token');
const wgIface = require('../utils/wg_iface');
const wgKeys = require('../utils/wg_keys');
const { meshCidrFor, meshAllowedIpsFor } = require('../utils/mesh_addressing');
const router = express.Router();
const IFACE = process.env.THETA_MESH_IFACE || 'wg-mesh';
const MESH_LISTEN_PORT = process.env.THETA_MESH_LISTEN_PORT || 51820;
async function ensureLocalIdentity() {
if (!conf.wireguard) conf.wireguard = {};
if (!conf.wireguard.serverPublicKey || !conf.wireguard.serverPrivateKey) {
// wg_bootstrap.js normally does this at startup; guard here too so this
// route works even if bootstrap hasn't run yet in a given environment.
const kp = wgKeys.generateKeypair();
conf.wireguard.serverPublicKey = kp.publicKey;
conf.wireguard.serverPrivateKey = kp.privateKey;
}
return conf.wireguard;
}
// Mint a single-use mesh join token — the credential a NEW gateway presents
// to register into THIS gateway's mesh.
router.post('/join-tokens', middleware.auth, middleware.requireJumpAdmin, async (req, res, next) => {
try {
const { token, expiresInSeconds } = await meshJoinToken.mint();
res.json({ status: 'ok', token, expiresInSeconds });
} catch (e) { next(e); }
});
// Called by a REMOTE gateway to register itself into THIS gateway's mesh.
// Bearer mesh join token, no admin session (service-to-service, same as
// theta-directory's POST /api/site/register-spoke pattern).
router.post('/register', async (req, res, next) => {
try {
const auth = req.headers.authorization || '';
const token = auth.startsWith('Bearer ') ? auth.slice(7).trim() : '';
if (!(await meshJoinToken.consume(token))) {
return res.status(401).json({ status: 'error', message: 'invalid or already-used mesh join token' });
}
const { publicKey, endpoint, siteSlug } = req.body || {};
if (!publicKey || !endpoint) {
return res.status(400).json({ status: 'error', message: 'publicKey and endpoint are required' });
}
const self = await ensureLocalIdentity();
const peer = await meshGateway.register({ publicKey, endpoint, siteSlug });
await wgIface.ensureInterface(IFACE);
wgIface.setPrivateKey(IFACE, self.serverPrivateKey, MESH_LISTEN_PORT);
// This gateway's own mesh index -- assigned to ITSELF the first time
// anyone registers with it, since a solo gateway has no index yet.
const ownIndex = await ensureOwnMeshIndex();
wgIface.setAddress(IFACE, meshCidrFor(ownIndex));
wgIface.setPeer(IFACE, {
publicKey: peer.publicKey,
endpoint: peer.endpoint,
allowedIPs: meshAllowedIpsFor(peer.meshIndex),
keepalive: 25
});
res.json({
status: 'ok',
meshIndex: peer.meshIndex,
gateway: {
publicKey: conf.wireguard.serverPublicKey,
endpoint: conf.wireguard.serverEndpoint || '',
meshIndex: ownIndex
}
});
} catch (e) { next(e); }
});
// This gateway's own mesh index is just "the lowest free index, stable once
// picked" -- stored as a synthetic self-entry in the same registry so it
// survives restarts the same way peer entries do.
async function ensureOwnMeshIndex() {
const self = await meshGateway.findByPublicKey(conf.wireguard.serverPublicKey);
if (self) return self.meshIndex;
const created = await meshGateway.register({
publicKey: conf.wireguard.serverPublicKey,
endpoint: conf.wireguard.serverEndpoint || '',
siteSlug: '(self)'
});
return created.meshIndex;
}
// Admin-initiated: join THIS gateway into a remote gateway's mesh. Generates
// (or reuses) this gateway's identity, brings up the local interface, calls
// the remote's /register, and applies the peer it gets back -- so after this
// call both sides have a live, working wg0 peer entry for each other.
router.post('/join', middleware.auth, middleware.requireJumpAdmin, async (req, res, next) => {
try {
const { remoteEndpoint, joinToken } = req.body || {};
if (!remoteEndpoint || !joinToken) {
return res.status(400).json({ status: 'error', message: 'remoteEndpoint and joinToken are required' });
}
const self = await ensureLocalIdentity();
await wgIface.ensureInterface(IFACE);
wgIface.setPrivateKey(IFACE, self.serverPrivateKey, MESH_LISTEN_PORT);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
let resp;
try {
resp = await fetch(String(remoteEndpoint).replace(/\/+$/, '') + '/api/mesh/register', {
method: 'POST',
headers: { Authorization: 'Bearer ' + joinToken, 'Content-Type': 'application/json' },
body: JSON.stringify({
publicKey: self.serverPublicKey,
endpoint: self.serverEndpoint || '',
siteSlug: process.env.SITE_SLUG || ''
}),
signal: controller.signal
});
} finally { clearTimeout(timer); }
if (!resp.ok) {
const text = (await resp.text().catch(() => '')).slice(0, 300);
return res.status(502).json({ status: 'error', message: 'remote registration failed: HTTP ' + resp.status + ' ' + text });
}
const data = await resp.json();
wgIface.setAddress(IFACE, meshCidrFor(data.meshIndex));
await meshGateway.register({ publicKey: data.gateway.publicKey, endpoint: data.gateway.endpoint, siteSlug: '(remote master)' });
wgIface.setPeer(IFACE, {
publicKey: data.gateway.publicKey,
endpoint: data.gateway.endpoint,
allowedIPs: meshAllowedIpsFor(data.gateway.meshIndex),
keepalive: 25
});
res.json({ status: 'ok', meshIndex: data.meshIndex, peerMeshIndex: data.gateway.meshIndex });
} catch (e) { next(e); }
});
router.get('/gateways', middleware.auth, middleware.requireJumpAdmin, async (req, res, next) => {
try {
const gateways = await meshGateway.list();
res.json({ status: 'ok', gateways, iface: IFACE, kernelWireguard: wgIface.kernelWireguardAvailable() });
} catch (e) { next(e); }
});
module.exports = router;
+27
View File
@@ -0,0 +1,27 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const { meshCidrFor, meshAllowedIpsFor, MAX_MESH_INDEX, MIN_MESH_INDEX } = require('../../utils/mesh_addressing');
test('meshCidrFor renders the .1 address in the site\'s /24', () => {
assert.equal(meshCidrFor(1), '172.24.1.1/24');
assert.equal(meshCidrFor(254), '172.24.254.1/24');
});
test('meshAllowedIpsFor covers both the mesh /24 and the site\'s 10.x/16', () => {
assert.deepEqual(meshAllowedIpsFor(5), ['172.24.5.0/24', '10.5.0.0/16']);
});
test('rejects index 0 and 255 (reserved) and the out-of-range/non-integer cases', () => {
assert.throws(() => meshCidrFor(0));
assert.throws(() => meshCidrFor(255));
assert.throws(() => meshCidrFor(MAX_MESH_INDEX + 1));
assert.throws(() => meshCidrFor(MIN_MESH_INDEX - 1));
assert.throws(() => meshCidrFor(1.5));
assert.throws(() => meshCidrFor('1'));
});
test('MAX_MESH_INDEX matches the documented 254-site ceiling', () => {
assert.equal(MAX_MESH_INDEX, 254);
});
+27
View File
@@ -0,0 +1,27 @@
'use strict';
// Mesh subnet addressing math (MULTI_SITE_SPEC.md Appendix A): one octet per
// site, 172.24.<idx>.0/16 + 10.<idx>.0.0/16, idx 1-254 (0/255 reserved).
// Pure/no I/O so it's cheaply unit-testable apart from routes/mesh.js.
const MESH_SUBNET_PREFIX = '172.24';
const MAX_MESH_INDEX = 254;
const MIN_MESH_INDEX = 1;
function meshCidrFor(meshIndex) {
assertValidIndex(meshIndex);
return `${MESH_SUBNET_PREFIX}.${meshIndex}.1/24`;
}
function meshAllowedIpsFor(meshIndex) {
assertValidIndex(meshIndex);
return [`${MESH_SUBNET_PREFIX}.${meshIndex}.0/24`, `10.${meshIndex}.0.0/16`];
}
function assertValidIndex(meshIndex) {
if (!Number.isInteger(meshIndex) || meshIndex < MIN_MESH_INDEX || meshIndex > MAX_MESH_INDEX) {
throw new Error(`mesh index must be an integer in [${MIN_MESH_INDEX}, ${MAX_MESH_INDEX}], got ${meshIndex}`);
}
}
module.exports = { meshCidrFor, meshAllowedIpsFor, MESH_SUBNET_PREFIX, MAX_MESH_INDEX, MIN_MESH_INDEX };
+29
View File
@@ -0,0 +1,29 @@
'use strict';
// Single-use, short-lived tokens that let a new theta-gateway register into
// this one's WireGuard mesh (POST /api/mesh/register). Same shape as
// theta-directory's site join keys: minted by an admin, shown once, GETDEL
// (Redis) on use so a token can register exactly one gateway, ever.
const crypto = require('crypto');
const conf = require('@simpleworkjs/conf');
const { getRedis } = require('../models/index');
const TTL_SECONDS = 15 * 60;
const key = (token) => `${conf.redis.prefix}mesh_join_token:${token}`;
async function mint() {
const token = 'mjt_' + crypto.randomBytes(24).toString('base64url');
const redis = await getRedis();
await redis.set(key(token), '1', { EX: TTL_SECONDS });
return { token, expiresInSeconds: TTL_SECONDS };
}
async function consume(token) {
if (!token) return false;
const redis = await getRedis();
return (await redis.getDel(key(token))) === '1';
}
module.exports = { mint, consume };
+156
View File
@@ -0,0 +1,156 @@
'use strict';
// Bring up a local WireGuard interface, preferring the in-kernel
// implementation and falling back to the userspace `wireguard-go` reference
// implementation when the kernel module isn't available (older/hardened
// kernels, some container/cloud images, non-Linux). Both paths end with an
// identically-named network interface that `wg`/`ip` commands (and the rest
// of this module) treat the same way -- callers never need to know which
// mode ended up in use.
const { execFileSync, spawn } = require('child_process');
const probes = new Map(); // name -> { mode, process (userspace only) }
function run(cmd, args) {
return execFileSync(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }).toString();
}
function tryRun(cmd, args) {
try { return { ok: true, out: run(cmd, args) }; }
catch (e) { return { ok: false, err: (e.stderr || e.message || '').toString() }; }
}
// One-time, cheap probe: can this kernel create a wireguard-type link at
// all? Uses a throwaway interface name so it never collides with a real one.
let kernelSupport = null;
function kernelWireguardAvailable() {
if (kernelSupport !== null) return kernelSupport;
const probeName = 'wgprobe' + process.pid;
const add = tryRun('ip', ['link', 'add', 'dev', probeName, 'type', 'wireguard']);
if (add.ok) {
tryRun('ip', ['link', 'del', 'dev', probeName]);
kernelSupport = true;
} else {
kernelSupport = false;
}
return kernelSupport;
}
function interfaceExists(name) {
return tryRun('ip', ['link', 'show', 'dev', name]).ok;
}
async function waitForInterface(name, timeoutMs = 5000) {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
if (interfaceExists(name)) return true;
await new Promise((r) => setTimeout(r, 100));
}
return false;
}
// Idempotent: calling this again for an interface that's already up (kernel
// or userspace) is a no-op, not an error.
async function ensureInterface(name) {
if (interfaceExists(name)) {
return { mode: probes.get(name) ? probes.get(name).mode : 'kernel' };
}
if (kernelWireguardAvailable()) {
const add = tryRun('ip', ['link', 'add', 'dev', name, 'type', 'wireguard']);
if (!add.ok && !/File exists/.test(add.err)) {
throw new Error(`kernel WireGuard interface creation failed: ${add.err}`);
}
probes.set(name, { mode: 'kernel' });
console.log(`[wg_iface] ${name}: using in-kernel WireGuard`);
return { mode: 'kernel' };
}
// Userspace fallback: wireguard-go daemonizes and creates the TUN device
// itself; we just wait for it to appear rather than assuming a fixed delay.
console.log(`[wg_iface] ${name}: kernel WireGuard unavailable, falling back to wireguard-go (userspace)`);
const child = spawn('wireguard-go', [name], { detached: true, stdio: 'ignore' });
child.unref();
const up = await waitForInterface(name);
if (!up) throw new Error(`wireguard-go did not bring up interface '${name}' within timeout`);
probes.set(name, { mode: 'userspace', pid: child.pid });
return { mode: 'userspace', pid: child.pid };
}
function setPrivateKey(name, privateKeyBase64, listenPort) {
// `wg setconf` reads the private key from a file, not argv (argv would leak
// it via /proc/<pid>/cmdline to anyone on the host). Pipe it through stdin
// via a temp file instead -- see setPeer's note on the same tradeoff.
// ListenPort matters: without one, WG binds an ephemeral port, which is
// fine for a purely outbound roaming client but useless for a gateway
// another gateway needs to dial back into as an Endpoint.
const fs = require('fs');
const os = require('os');
const path = require('path');
const tmp = path.join(os.tmpdir(), `wg-${name}-${Date.now()}.conf`);
const lines = ['[Interface]', `PrivateKey = ${privateKeyBase64}`];
if (listenPort) lines.push(`ListenPort = ${listenPort}`);
fs.writeFileSync(tmp, lines.join('\n') + '\n', { mode: 0o600 });
try {
run('wg', ['setconf', name, tmp]);
} finally {
fs.unlinkSync(tmp);
}
}
function setAddress(name, cidr) {
// Flush first so re-applying (e.g. after a mesh index reassignment, which
// shouldn't normally happen but must not silently stack addresses if it
// does) leaves exactly one address, not an accumulating list.
tryRun('ip', ['addr', 'flush', 'dev', name]);
run('ip', ['addr', 'add', cidr, 'dev', name]);
run('ip', ['link', 'set', 'up', 'dev', name]);
}
// Apply (or update) one peer. Safe to call repeatedly for the same peer --
// `wg set ... peer <pub>` upserts.
//
// `wg set ... allowed-ips` ONLY configures WireGuard's own crypto-routing
// table (which packets get encrypted/decrypted for this peer) -- it does
// NOT add a kernel route for that destination. wg-quick does that as a
// separate step; since we drive `wg`/`ip` directly (no wg-quick), we have to
// add it ourselves or the tunnel handshakes fine but nothing ever actually
// routes through it (confirmed the hard way: a real encrypted handshake
// completed between two containers with zero kernel route present, and
// ping still showed 100% loss).
function setPeer(name, { publicKey, endpoint, allowedIPs, keepalive }) {
const ips = allowedIPs || [];
const args = ['set', name, 'peer', publicKey, 'allowed-ips', ips.join(',')];
if (endpoint) args.push('endpoint', endpoint);
if (keepalive) args.push('persistent-keepalive', String(keepalive));
run('wg', args);
for (const cidr of ips) {
const add = tryRun('ip', ['route', 'add', cidr, 'dev', name]);
// "File exists" happens when the interface's own /24 already covers
// this range (added automatically by `ip addr add`) -- fine, not an
// error. Anything else should surface.
if (!add.ok && !/File exists/.test(add.err)) {
throw new Error(`failed to add kernel route ${cidr} via ${name}: ${add.err}`);
}
}
}
// TODO: doesn't clean up the kernel routes setPeer added for this peer's
// AllowedIPs (would need to record or query them first) -- not exercised by
// any caller yet (nothing in this codebase removes a mesh peer today), but
// flagging so whoever adds that doesn't get bitten by stale routes.
function removePeer(name, publicKey) {
tryRun('wg', ['set', name, 'peer', publicKey, 'remove']);
}
module.exports = {
kernelWireguardAvailable,
ensureInterface,
setPrivateKey,
setAddress,
setPeer,
removePeer,
interfaceExists
};