feat(mesh): real gateway-to-gateway WireGuard mesh (site-to-site tunnels)
The existing WireGuard code (models/wg_site.js, routes/wireguard.js) is the roaming-client/exit-node feature -- individual peer configs an admin hands out, not gateway-to-gateway mesh peering. This adds the latter, per MULTI_SITE_SPEC.md §4: two theta-gateway instances mesh by one calling the other's POST /api/mesh/register with a join token (minted via POST /api/mesh/join-tokens, admin-gated); both sides end up with a live wg0 peer for the other, mesh-indexed per Appendix A's addressing (172.24.<idx>.0/16 + 10.<idx>.0.0/16, idx 1-254). - utils/wg_iface.js: brings up the local interface, preferring in-kernel WireGuard (ip link add type wireguard) and falling back to userspace wireguard-go when the kernel module isn't available. Both packages added to the Dockerfile. - utils/mesh_addressing.js: pure addressing math, unit tested (test/unit/mesh_addressing.test.js). - models/mesh_gateway.js: Redis-backed registry of known peer gateways (same pattern as wg_site.js), assigns + persists mesh indexes. - utils/mesh_join_token.js: single-use bootstrap credential, same GETDEL-on-Redis pattern already used on the theta-directory side. - routes/mesh.js: /join-tokens (admin), /register (bearer token, no session -- called by a remote gateway), /join (admin, initiates from this side), /gateways (admin, list). Verified with a REAL two-container test (not mocked): two independent containers, each running this actual code, meshed via a live join-token handshake, brought up real kernel WireGuard interfaces, and passed ICMP traffic across the resulting encrypted tunnel end to end (0% packet loss). That test caught a real bug worth calling out: `wg set ... peer ... allowed-ips` only configures WireGuard's own crypto-routing table -- it does NOT add a kernel route for that destination (wg-quick normally does this as a separate step; we don't use wg-quick). A real encrypted handshake completed between the two containers with the route missing, and ping still showed 100% loss until setPeer() was fixed to add the corresponding `ip route add <allowed-ip> dev <iface>` itself.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
'use strict';
|
||||
|
||||
// Mesh subnet addressing math (MULTI_SITE_SPEC.md Appendix A): one octet per
|
||||
// site, 172.24.<idx>.0/16 + 10.<idx>.0.0/16, idx 1-254 (0/255 reserved).
|
||||
// Pure/no I/O so it's cheaply unit-testable apart from routes/mesh.js.
|
||||
|
||||
const MESH_SUBNET_PREFIX = '172.24';
|
||||
const MAX_MESH_INDEX = 254;
|
||||
const MIN_MESH_INDEX = 1;
|
||||
|
||||
function meshCidrFor(meshIndex) {
|
||||
assertValidIndex(meshIndex);
|
||||
return `${MESH_SUBNET_PREFIX}.${meshIndex}.1/24`;
|
||||
}
|
||||
|
||||
function meshAllowedIpsFor(meshIndex) {
|
||||
assertValidIndex(meshIndex);
|
||||
return [`${MESH_SUBNET_PREFIX}.${meshIndex}.0/24`, `10.${meshIndex}.0.0/16`];
|
||||
}
|
||||
|
||||
function assertValidIndex(meshIndex) {
|
||||
if (!Number.isInteger(meshIndex) || meshIndex < MIN_MESH_INDEX || meshIndex > MAX_MESH_INDEX) {
|
||||
throw new Error(`mesh index must be an integer in [${MIN_MESH_INDEX}, ${MAX_MESH_INDEX}], got ${meshIndex}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { meshCidrFor, meshAllowedIpsFor, MESH_SUBNET_PREFIX, MAX_MESH_INDEX, MIN_MESH_INDEX };
|
||||
@@ -0,0 +1,29 @@
|
||||
'use strict';
|
||||
|
||||
// Single-use, short-lived tokens that let a new theta-gateway register into
|
||||
// this one's WireGuard mesh (POST /api/mesh/register). Same shape as
|
||||
// theta-directory's site join keys: minted by an admin, shown once, GETDEL
|
||||
// (Redis) on use so a token can register exactly one gateway, ever.
|
||||
|
||||
const crypto = require('crypto');
|
||||
const conf = require('@simpleworkjs/conf');
|
||||
const { getRedis } = require('../models/index');
|
||||
|
||||
const TTL_SECONDS = 15 * 60;
|
||||
|
||||
const key = (token) => `${conf.redis.prefix}mesh_join_token:${token}`;
|
||||
|
||||
async function mint() {
|
||||
const token = 'mjt_' + crypto.randomBytes(24).toString('base64url');
|
||||
const redis = await getRedis();
|
||||
await redis.set(key(token), '1', { EX: TTL_SECONDS });
|
||||
return { token, expiresInSeconds: TTL_SECONDS };
|
||||
}
|
||||
|
||||
async function consume(token) {
|
||||
if (!token) return false;
|
||||
const redis = await getRedis();
|
||||
return (await redis.getDel(key(token))) === '1';
|
||||
}
|
||||
|
||||
module.exports = { mint, consume };
|
||||
@@ -0,0 +1,156 @@
|
||||
'use strict';
|
||||
|
||||
// Bring up a local WireGuard interface, preferring the in-kernel
|
||||
// implementation and falling back to the userspace `wireguard-go` reference
|
||||
// implementation when the kernel module isn't available (older/hardened
|
||||
// kernels, some container/cloud images, non-Linux). Both paths end with an
|
||||
// identically-named network interface that `wg`/`ip` commands (and the rest
|
||||
// of this module) treat the same way -- callers never need to know which
|
||||
// mode ended up in use.
|
||||
|
||||
const { execFileSync, spawn } = require('child_process');
|
||||
|
||||
const probes = new Map(); // name -> { mode, process (userspace only) }
|
||||
|
||||
function run(cmd, args) {
|
||||
return execFileSync(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }).toString();
|
||||
}
|
||||
|
||||
function tryRun(cmd, args) {
|
||||
try { return { ok: true, out: run(cmd, args) }; }
|
||||
catch (e) { return { ok: false, err: (e.stderr || e.message || '').toString() }; }
|
||||
}
|
||||
|
||||
// One-time, cheap probe: can this kernel create a wireguard-type link at
|
||||
// all? Uses a throwaway interface name so it never collides with a real one.
|
||||
let kernelSupport = null;
|
||||
function kernelWireguardAvailable() {
|
||||
if (kernelSupport !== null) return kernelSupport;
|
||||
const probeName = 'wgprobe' + process.pid;
|
||||
const add = tryRun('ip', ['link', 'add', 'dev', probeName, 'type', 'wireguard']);
|
||||
if (add.ok) {
|
||||
tryRun('ip', ['link', 'del', 'dev', probeName]);
|
||||
kernelSupport = true;
|
||||
} else {
|
||||
kernelSupport = false;
|
||||
}
|
||||
return kernelSupport;
|
||||
}
|
||||
|
||||
function interfaceExists(name) {
|
||||
return tryRun('ip', ['link', 'show', 'dev', name]).ok;
|
||||
}
|
||||
|
||||
async function waitForInterface(name, timeoutMs = 5000) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (interfaceExists(name)) return true;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Idempotent: calling this again for an interface that's already up (kernel
|
||||
// or userspace) is a no-op, not an error.
|
||||
async function ensureInterface(name) {
|
||||
if (interfaceExists(name)) {
|
||||
return { mode: probes.get(name) ? probes.get(name).mode : 'kernel' };
|
||||
}
|
||||
|
||||
if (kernelWireguardAvailable()) {
|
||||
const add = tryRun('ip', ['link', 'add', 'dev', name, 'type', 'wireguard']);
|
||||
if (!add.ok && !/File exists/.test(add.err)) {
|
||||
throw new Error(`kernel WireGuard interface creation failed: ${add.err}`);
|
||||
}
|
||||
probes.set(name, { mode: 'kernel' });
|
||||
console.log(`[wg_iface] ${name}: using in-kernel WireGuard`);
|
||||
return { mode: 'kernel' };
|
||||
}
|
||||
|
||||
// Userspace fallback: wireguard-go daemonizes and creates the TUN device
|
||||
// itself; we just wait for it to appear rather than assuming a fixed delay.
|
||||
console.log(`[wg_iface] ${name}: kernel WireGuard unavailable, falling back to wireguard-go (userspace)`);
|
||||
const child = spawn('wireguard-go', [name], { detached: true, stdio: 'ignore' });
|
||||
child.unref();
|
||||
const up = await waitForInterface(name);
|
||||
if (!up) throw new Error(`wireguard-go did not bring up interface '${name}' within timeout`);
|
||||
probes.set(name, { mode: 'userspace', pid: child.pid });
|
||||
return { mode: 'userspace', pid: child.pid };
|
||||
}
|
||||
|
||||
function setPrivateKey(name, privateKeyBase64, listenPort) {
|
||||
// `wg setconf` reads the private key from a file, not argv (argv would leak
|
||||
// it via /proc/<pid>/cmdline to anyone on the host). Pipe it through stdin
|
||||
// via a temp file instead -- see setPeer's note on the same tradeoff.
|
||||
// ListenPort matters: without one, WG binds an ephemeral port, which is
|
||||
// fine for a purely outbound roaming client but useless for a gateway
|
||||
// another gateway needs to dial back into as an Endpoint.
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const tmp = path.join(os.tmpdir(), `wg-${name}-${Date.now()}.conf`);
|
||||
const lines = ['[Interface]', `PrivateKey = ${privateKeyBase64}`];
|
||||
if (listenPort) lines.push(`ListenPort = ${listenPort}`);
|
||||
fs.writeFileSync(tmp, lines.join('\n') + '\n', { mode: 0o600 });
|
||||
try {
|
||||
run('wg', ['setconf', name, tmp]);
|
||||
} finally {
|
||||
fs.unlinkSync(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
function setAddress(name, cidr) {
|
||||
// Flush first so re-applying (e.g. after a mesh index reassignment, which
|
||||
// shouldn't normally happen but must not silently stack addresses if it
|
||||
// does) leaves exactly one address, not an accumulating list.
|
||||
tryRun('ip', ['addr', 'flush', 'dev', name]);
|
||||
run('ip', ['addr', 'add', cidr, 'dev', name]);
|
||||
run('ip', ['link', 'set', 'up', 'dev', name]);
|
||||
}
|
||||
|
||||
// Apply (or update) one peer. Safe to call repeatedly for the same peer --
|
||||
// `wg set ... peer <pub>` upserts.
|
||||
//
|
||||
// `wg set ... allowed-ips` ONLY configures WireGuard's own crypto-routing
|
||||
// table (which packets get encrypted/decrypted for this peer) -- it does
|
||||
// NOT add a kernel route for that destination. wg-quick does that as a
|
||||
// separate step; since we drive `wg`/`ip` directly (no wg-quick), we have to
|
||||
// add it ourselves or the tunnel handshakes fine but nothing ever actually
|
||||
// routes through it (confirmed the hard way: a real encrypted handshake
|
||||
// completed between two containers with zero kernel route present, and
|
||||
// ping still showed 100% loss).
|
||||
function setPeer(name, { publicKey, endpoint, allowedIPs, keepalive }) {
|
||||
const ips = allowedIPs || [];
|
||||
const args = ['set', name, 'peer', publicKey, 'allowed-ips', ips.join(',')];
|
||||
if (endpoint) args.push('endpoint', endpoint);
|
||||
if (keepalive) args.push('persistent-keepalive', String(keepalive));
|
||||
run('wg', args);
|
||||
|
||||
for (const cidr of ips) {
|
||||
const add = tryRun('ip', ['route', 'add', cidr, 'dev', name]);
|
||||
// "File exists" happens when the interface's own /24 already covers
|
||||
// this range (added automatically by `ip addr add`) -- fine, not an
|
||||
// error. Anything else should surface.
|
||||
if (!add.ok && !/File exists/.test(add.err)) {
|
||||
throw new Error(`failed to add kernel route ${cidr} via ${name}: ${add.err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: doesn't clean up the kernel routes setPeer added for this peer's
|
||||
// AllowedIPs (would need to record or query them first) -- not exercised by
|
||||
// any caller yet (nothing in this codebase removes a mesh peer today), but
|
||||
// flagging so whoever adds that doesn't get bitten by stale routes.
|
||||
function removePeer(name, publicKey) {
|
||||
tryRun('wg', ['set', name, 'peer', publicKey, 'remove']);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
kernelWireguardAvailable,
|
||||
ensureInterface,
|
||||
setPrivateKey,
|
||||
setAddress,
|
||||
setPeer,
|
||||
removePeer,
|
||||
interfaceExists
|
||||
};
|
||||
Reference in New Issue
Block a user