Files
proxy/nodejs/models/host.js
T
wmantly 4321826dc8 feat: Add multi-target load balancing support
Fixes #47.
- Added lua-resty-balancer to dependencies (Dockerfile & install.sh).
- Added 'targets' field to the Host model to hold additional targets.
- Updated the UI to allow inputting additional targets (IP:port).
- Updated targetinfo.lua to parse the additional targets and load balance between them and the primary target using resty.balancer.round_robin.
2026-07-21 00:48:43 -04:00

643 lines
22 KiB
JavaScript
Executable File

'use strict';
const Table = require('.');
const {Domain} = require('.').models;
const {getCert, setCert, deleteCert} = require('./cert');
const ModelPs = require('../utils/model_pubsub');
const tldExtract = require('tld-extract').parse_host;
const LetsEncrypt = require('../utils/letsencrypt');
const conf = require('@simpleworkjs/conf');
const letsEncrypt = new LetsEncrypt({
directoryUrl: conf.environment === "production" ?
LetsEncrypt.AcmeClient.directory.letsencrypt.production :
LetsEncrypt.AcmeClient.directory.letsencrypt.staging,
});
class Host extends Table{
static _key = 'host';
static _keyMap = {
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
// min 1 so wildcard patterns like "**" / "*" are allowed (see
// utils/hostname_validate.js; format is enforced at the route layer).
'host': {isRequired: true, type: 'string', min: 1, max: 500},
'ip': {isRequired: true, type: 'string', min: 3, max: 500},
'targetPort': {isRequired: true, type: 'number', min:0, max:65535},
'targets': {default: function(){return []}, isRequired: false, type: 'object'},
'forcessl': {isRequired: false, default: true, type: 'boolean'},
'targetssl': {isRequired: false, default: false, type: 'boolean'},
'is_cache': {default: false, isRequired: false, type: 'boolean',},
// Per-host reverse-proxy controls. Enforced in OpenResty by
// ops/nginx_conf/hostfeatures.lua, which reads these straight off the
// Redis hash. Object fields are JSON-encoded by model-redis.
'ratelimit_enabled': {default: false, isRequired: false, type: 'boolean',},
'ratelimit_rate': {default: 10, isRequired: false, type: 'number', min: 1, max: 1000000},
'ratelimit_burst': {default: 20, isRequired: false, type: 'number', min: 0, max: 1000000},
'respcache_enabled': {default: false, isRequired: false, type: 'boolean',},
'hsts_enabled': {default: false, isRequired: false, type: 'boolean',},
// Per-host HTTP basic auth. basicauth_users is {username: base64(sha1(pw))}
// (hashed at the route layer, see utils/basicauth.js); enforced in
// ops/nginx_conf/hostfeatures.lua.
'basicauth_enabled': {default: false, isRequired: false, type: 'boolean',},
'basicauth_realm': {default: 'Restricted', isRequired: false, type: 'string', min: 1, max: 128},
'basicauth_users': {default: function(){return {}}, isRequired: false, type: 'object',},
// Per-host SSO (OIDC via conf.oidc) — enforced by a signed session cookie
// checked in ops/nginx_conf/hostfeatures.lua. Empty allow-lists mean "any
// authenticated user". basic auth and SSO are OR'd (either satisfies).
'sso_enabled': {default: false, isRequired: false, type: 'boolean',},
'sso_allow_users': {default: function(){return []}, isRequired: false, type: 'object',},
'sso_allow_groups': {default: function(){return []}, isRequired: false, type: 'object',},
'req_headers': {default: function(){return {}}, isRequired: false, type: 'object',},
'resp_headers': {default: function(){return {}}, isRequired: false, type: 'object',},
'ip_allow': {default: function(){return []}, isRequired: false, type: 'object',},
'ip_deny': {default: function(){return []}, isRequired: false, type: 'object',},
'is_wildcard': {default: false, isRequired: false, type: 'boolean',},
'wildcard_status': {isRequired: false, type: 'string', min: 3, max: 500},
'wildcard_matchAny': {default: false, isRequired: false, type: 'boolean',},
'wildcard_parent': {isRequired: false, type: 'string', min: 3, max: 500},
'wildcard_expires': {isRequired: false, type: 'number'},
'domain': {model: 'Domain', rel: 'one'},
}
static lookUpObj = {};
static __lookUpIsReady = false;
static async addCache(host, parentOBJ){
try{
parentOBJ = await this.get(parentOBJ.host);
if(parentOBJ.is_cache){
return;
}
// Give the on-demand cache entry a TTL so it auto-expires instead of
// living forever. Only the record hash carries the TTL (model-redis
// reaps the dangling index member on the next read), so OpenResty's
// direct HGETALL sees a miss once it expires and re-resolves through
// this lookup path. 0/falsy conf disables expiry.
let ttl = conf.cacheTTL > 0 ? {ttl: conf.cacheTTL} : undefined;
await this.create({
...parentOBJ,
host: host,
is_cache: true,
is_wildcard: false,
wildcard_parent: parentOBJ.host
}, ttl);
await Cached.create({
host: host,
parent: parentOBJ.host
}, ttl);
}catch(error){
console.error('add cache error', {...parentOBJ, host, is_cache: true}, error);
throw error;
}
}
async bustCache(parent){
try{
let cached = await Cached.listDetail();
for(let cache of cached){
if(cache.parent == parent){
let host = await Host.get(cache.host);
await this.remove.apply(host);
await cache.remove();
}
}
}catch(error){
console.error('bust cache error', error)
// throw error;
}
}
// Remove every cached host entry regardless of its parent. Cache entries are
// the `is_cache` hosts created on demand by addCache() for wildcard subdomain
// lookups; clearing them forces the next request for each subdomain to be
// resolved fresh through the lookup tree / host_lookup service.
static async clearCache(){
let count = 0;
try{
for(let cache of await Cached.listDetail()){
try{
let host = await Host.get(cache.host);
if(host && host.is_cache) await host.remove();
await cache.remove();
count++;
}catch(error){
console.error('clear cache entry error', cache.host, error);
}
}
await this.buildLookUpObj();
}catch(error){
console.error('clear cache error', error);
throw error;
}
return count;
}
static async create(data, ...args){
try{
// Validate requested host is valid host and domain
if(data.challengeType === 'DNS-01-wildcard'){
await this.validateWildcardCreate(data, args);
data.is_wildcard = true;
data.wildcard_status = "Starting"
}
// Validate requested host has a valid wildcard parent
if(data.challengeType === 'wildcardChild'){
let parentHost = await this.lookUp(data.host);
if(parentHost.is_wildcard){
data.wildcard_parent = parentHost.host;
}else{
throw new Error(`No parent wild card for ${data.host}`);
}
}
// Create the new host entry
let out = await super.create(data, ...args);
// Update the lookup table to reflect new host
await this.buildLookUpObj();
// Fire the request for the wild card cert
// This is "back ground" job, await is intentionally missing
if(data.challengeType === 'DNS-01-wildcard') out.createWildcardCert();
return out;
} catch(error){
throw error;
}
}
static async validateWildcardCreate(data, ...args){
console.log('validateWildcardCreate here')
try{
if(!data.host.startsWith('*.')) throw new Error('not wild card');
await Domain.get(data.host);
}catch(error){
console.log('validateWildcardCreate error', error)
if(error.status === 404) error.message = "No matching DNS provider registered"
throw new this.errors.ObjectValidateError([{key: 'host', message: error.message}]);
}
}
async createWildcardCert(){
console.log('createWildcardCert', this.domain)
if(!this.host.startsWith('*.')) throw new Error('not wild card');
try{
let host = this;
await host.update({
wildcard_status: 'Requesting',
});
let cert = await letsEncrypt.dnsWildcard(this.host, {
challengeCreateFn: async (authz, challenge, keyAuthorization) => {
await host.update({
wildcard_status: `Adding record`
});
try{
let parts = tldExtract(authz.identifier.value);
let res = await host.domain.createRecord(
{
type:'TXT',
name: `_acme-challenge${parts.sub ? `.${parts.sub}` : ''}`,
data: `${keyAuthorization}`
},
true // Force the record creation, even if the record exists
);
}catch(error){
console.log('model Host challengeCreateFn error:', error)
await host.update({
wildcard_status: `Add DNS record failed`
});
}
},
onDnsCheck: async(authz, checkCount)=>{
await host.update({
wildcard_status: `${checkCount} Checking DNS`
});
},
onDnsCheckFail: async(authz, error)=>{
await host.update({
wildcard_status: `DNS check failed for ${authz.identifier.value}`
});
},
onDnsCheckFound: async(authz)=>{
await host.update({
wildcard_status: `DNS check found`
});
},
onDnsCheckSuccess: async(authz)=>{
await host.update({
wildcard_status: `DNS check success`
})
},
onDnsCheckRemove: async(authz)=>{
await host.update({
wildcard_status: `DNS remove record`
})
},
challengeRemoveFn: async (authz, challenge, keyAuthorization)=>{
await host.update({
wildcard_status: `DNS remove record`
})
try{
let parts = tldExtract(authz.identifier.value);
await host.domain.deleteRecords(
{
type:'TXT',
name: `_acme-challenge${parts.sub ? `.${parts.sub}` : ''
}`,
content: `${keyAuthorization}`}
);
}catch(error){
await host.update({
wildcard_status: `DNS remove record failed for ${authz.identifier.value}`
})
}
},
});
let toAdd = {
cert_pem: cert.cert.split('\n\n')[0],
fullchain_pem: cert.cert,
privkey_pem: cert.key.toString(),
csr_pem: cert.csr.toString(),
expiry: 4120307657,
real_expiry: +LetsEncrypt.AcmeClient.crypto.readCertificateInfo(cert.cert).notAfter/1000,
}
await this.constructor.redisClient.SET(`${this.host}:latest`, JSON.stringify(toAdd));
await this.update({
wildcard_status: `Done`,
wildcard_expires: toAdd.real_expiry*1000,
});
return this;
}catch(error){
console.log('le failed', error)
this.update({
wildcard_status: `LE failed`
});
}
}
async checkWildcardForRenew(){
try{
if(this.is_wildcard && Date.now() > this.wildcard_expires - (30 * 24 * 60 * 60 * 1000)){
this.createWildcardCert();
}
}catch(error){
console.error('checkWildcardForRenew instance', this.host, error)
throw error;
}
}
static async checkWildcardForRenew(){
try{
for(let host of await this.listDetail()){
host.checkWildcardForRenew();
}
}catch(error){
console.error('checkWildcardForRenew', error)
throw error;
}
}
async update(data, ...args){
try{
// Mirror Host.create()'s challengeType handling (lines above) so an
// existing HTTP-01 host can be attached to a parent wildcard's cert
// after creation -- previously this was silently dropped since only
// create() understood challengeType, leaving no way to convert an
// existing host onto a wildcard once one was issued.
if(data && data.challengeType === 'wildcardChild'){
// Not Host.lookUp() -- this.host already has its own leaf in the
// tree (it already exists), so a plain lookUp() would just find
// itself. lookUpWildcardParent() checks the sibling "*" slot
// instead. See its comment for why create()'s own wildcardChild
// branch doesn't need this (a host being newly created hasn't
// claimed its own leaf yet, so plain lookUp() already falls
// through to the wildcard correctly there).
let parentHost = Host.lookUpWildcardParent(this.host);
if(parentHost && parentHost.is_wildcard){
data.wildcard_parent = parentHost.host;
}else{
throw new Error(`No parent wild card for ${this.host}`);
}
}
// Real hostname rename. model-redis's own update() (see super.update()
// below) already handles the Redis primary-key RENAME + collision
// check, and Host.buildLookUpObj() below already rebuilds the lookup
// tree afterward -- but the cert cache (models/cert.js, `${host}:latest`)
// is a separate record keyed by hostname string that the generic field
// system doesn't know about, so it doesn't move on its own. Only
// wildcard hosts (createWildcardCert) ever populate this key -- for a
// plain HTTP-01 host this is a no-op (nothing to migrate; auto-ssl
// transparently issues a fresh cert under the new name on first
// access, same as it does for any newly-created host).
let oldHost = this.host;
let renaming = data && typeof data.host === 'string' && data.host !== oldHost;
if(renaming){
let cert = await getCert(oldHost);
if(cert && Object.keys(cert).length) await setCert(data.host, cert);
}
let out = await super.update(data, ...args)
await this.bustCache(this.host);
await Host.buildLookUpObj();
if(renaming){
await deleteCert(oldHost);
// Work around a model-redis bug (as of ^1.5.0): super.update()'s
// field-application loop iterates _keyMap's definition order and
// only reassigns this[_key] (this.host) to the NEW value once it
// reaches the `host` field itself -- but `updated_on` (always:
// true, so always included) is defined BEFORE `host` in _keyMap,
// so it gets HSET while this.host is still the OLD name. Redis's
// HSET on a non-existent key (the old hash, just RENAMEd away)
// silently recreates it -- leaving a stray, incomplete hash under
// the old hostname that makes Host.exists(oldHost) wrongly return
// true forever, blocking that name from ever being reused.
await this.constructor.redisClient.DEL(`${conf.redis.prefix || ''}Host_${oldHost}`);
}
return out;
} catch(error){
throw error;
}
}
async remove(...args){
try{
let out = await super.remove(...args);
await Host.buildLookUpObj();
await this.bustCache(this.host);
await deleteCert(this.domain);
return out;
} catch(error){
throw error;
}
}
static async buildLookUpObj(){
/*
Build a look up tree for domain records in the redis back end to allow
complex looks with wildcards.
*/
// Build into a fresh, local tree instead of mutating the live one.
// buildLookUpObj is async (it awaits a redis get() per host) and runs on
// every host create/update/remove. If we wiped and repopulated the live
// this.lookUpObj in place, any concurrent lookUp() — which is called
// synchronously by the host_lookup service and never waits for readiness —
// would resolve against a half-built tree and randomly miss defined hosts.
// We only swap the completed tree in at the very end, so lookUp() always
// sees a complete tree (either the previous one or the new one).
let lookUpObj = {};
try{
// Loop over all the hosts in the redis.
for(let host of await this.list()){
// Spit the hosts on "." into its fragments .
let fragments = host.split('.');
// Hold a pointer to the root of the lookup tree.
let pointer = lookUpObj;
// Walk over each fragment, popping from right to left.
while(fragments.length){
let fragment = fragments.pop();
// Add a branch to the lookup at the current position
if(!pointer[fragment]){
pointer[fragment] = {};
}
// Add the record(leaf) when we hit the a full host name.
// #record denotes a leaf node on this tree.
if(fragments.length === 0){
pointer[fragment]['#record'] = await this.get(host)
// A single-level wildcard's issued cert also covers its own
// base domain (createWildcardCert requests altNames:
// [domain, *.domain] -- see utils/letsencrypt.js), but the
// base domain sits one level ABOVE the wildcard's own leaf
// in this tree (e.g. "*.cool.mysite.com" is a child of the
// node for "cool.mysite.com"). Without this, looking up the
// bare base domain when it has no host of its own falls
// through to nothing, even though the already-issued cert
// covers it. `pointer` here is still that parent node
// (reassigned to the child only below) -- stamp it too, but
// only if a real, explicitly-created host at that exact
// name hasn't already claimed this leaf (order-independent:
// this only ever fills a gap -- a real host's own pass
// through this loop always overwrites #record
// unconditionally when it's finalized, see above).
if(fragment === '*' && !pointer['#record']){
pointer['#record'] = pointer[fragment]['#record'];
}
}
// Advance the pointer to the next level of the tree.
pointer = pointer[fragment];
}
}
// Atomically publish the completed tree and mark lookUp ready.
this.lookUpObj = lookUpObj;
this.__lookUpIsReady = true;
}catch(error){
console.error(error);
}
}
static lookUp(host){
/*
Perform a complex lookup of @host on the look up tree.
*/
// Hold a pointer to the root of the look up tree
let place = this.lookUpObj;
// Hold the last passed long wild card.
let last_resort = {};
// Hold the parent element
let parent = undefined;
// Walk over each fragment of the host, from right to left
for(let fragment of host.split('.').reverse()){
parent = place;
// If a long wild card is found on this level, hold on to it
if(place['**']) last_resort = place['**'];
// If we have a match for the current fragment, update the current pointer
// A match in the lookup tree takes priority being a more exact match.
if({...last_resort, ...place}[fragment]){
place = {...last_resort, ...place}[fragment];
// If we have a not exact fragment match, a wild card will do.
}else if(place['*']){
place = place['*']
// If no fragment can be matched, continue with the long wild card branch.
}else if(last_resort){
place = last_resort;
}
}
// After the tree has been traversed, see if we have leaf node to return.
if(place && place['#record']) return place['#record'];
// If the parent has a wild, its the wildcard we want.
if(parent && parent['*'] && parent['*']['#record']) return parent['*']['#record'];
}
// Find the wildcard that could cover @host, regardless of whether @host is
// already registered as its own host. Unlike lookUp(), which walks to and
// returns @host's own exact-match leaf when one exists, this keeps looking
// for a sibling/child "*" slot, so it still finds the parent wildcard even
// when @host already has its own (non-wildcard) record. Used when attaching
// an already-created host to a wildcard after the fact (see update() below);
// Host.create()'s own wildcardChild handling can keep using plain lookUp()
// since a host being newly created hasn't claimed its own leaf yet.
//
// Two tree positions qualify, and we must check BOTH:
// 1. Child "*" of @host's own node -- @host is the wildcard's base domain
// (e.g. "*.cool.mysite.com" covers host="cool.mysite.com").
// 2. Sibling "*" one level up -- @host is a single-label subdomain of the
// wildcard (e.g. "*.nl.wgnode.com" covers host="sso.nl.wgnode.com").
// Case 2 is the common one and was previously missed: the walk consumed the
// leftmost label ("sso") and only inspected that leaf's "*" child, so an
// already-existing sibling subdomain could never be attached to its wildcard.
static lookUpWildcardParent(host){
let place = this.lookUpObj;
let parent = undefined;
for(let fragment of host.split('.').reverse()){
// @host may have no leaf of its own (brand-new subdomain); that case
// is already handled by plain lookUp()'s wildcard fallback in the
// caller, so just stop -- we've still tracked `parent` for case 2.
if(!place[fragment]){ place = undefined; break; }
parent = place;
place = place[fragment];
}
// Case 1: wildcard is a child of @host's own node.
if(place && place['*'] && place['*']['#record']) return place['*']['#record'];
// Case 2: wildcard is a sibling of @host's leftmost label.
if(parent && parent['*'] && parent['*']['#record']) return parent['*']['#record'];
}
static async lookUpReady(){
/*
Wait for the lookup tree to be built.
*/
// Check every 5ms to see if the look up tree is ready
while(!this.__lookUpIsReady) await new Promise(r => setTimeout(r, 5));
return true;
}
}
Host.register(ModelPs(Host))
class Cached extends Table{
static _key = 'host';
static _keyMap = {
'host': {isRequired: true, type: 'string', min: 3, max: 500},
'parent': {isRequired: true, type: 'string', min: 3, max: 500},
}
}
(async function(){
await Host.buildLookUpObj();
})();
module.exports = {Host: ModelPs(Host)};
if(require.main === module){(async function(){
try{
await Host.lookUpReady();
let host = await Host.get('*.new.test.wtf')
console.log('host', host.domain.provider.api);
// let res = await Host.create({
// host: '*.test.holycore.quest',
// ip: '192.168.1.47',
// 'created_by': 'william',
// 'targetPort': 8006,
// 'forcessl': false,
// 'targetssl': true,
// 'is_wildcard': true,
// })
// console.log('IIFE res:\n', res)
// console.log(Host.test(55))
// console.log(await Host.list())
// console.log(await Cached.listDetail())
// console.log('IIFE lookup:', Host.lookUp('bld3324sdf.test.holycore.quest'))
// console.log(Host.lookUpObj)
// console.log(await Host.listDetail())
// // console.log(Host.lookUpObj['com']['vm42'])
// // console.log('test-res', await Host.lookUp('payments.718it.biz'))
// let count = 6
// console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz')
// console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined)
// console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**')
// console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com')
// console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com')
// console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com')
// console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com')
// console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**')
// console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz')
// console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined)
// console.log(count++, Host.lookUp('blah.biz') === undefined)
// console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net')
// console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com')
// console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com')
// console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**')
// console.log(count++, Host.lookUp('718it.biz').host === '718it.biz')
}catch(error){
console.log('IIFE test area error:', error)
}
})()}