theta42/proxy
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #100 from theta42/model-redis

Browse Source 
Model redis
This commit is contained in:
William Mantly
2026-07-09 22:20:53 -04:00
committed by GitHub
parent 5309719622 398d2d9979
commit 655aeb47a2
20 changed files with 545 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/conf/base.js
+10 -1
View File
@@ -14,5 +14,14 @@ module.exports = {
socketFile: '/var/run/proxy_lookup.socket',
redis: {
prefix: 'proxy_'
}
},
service:{
hostScheduler:{
enabled: true,
initial: 30000,
interval: 86400000,
}
},
};
nodejs/conf/development.js
+25
View File
@@ -0,0 +1,25 @@
'use strict';
// Using https://github.com/simpleworkjs/conf to handle configuration
module.exports = {
userModel: 'redis', // pam, redis, ldap
ldap: {
url: 'ldap://192.168.1.55:389',
bindDN: 'cn=ldapclient service,ou=people,dc=theta42,dc=com',
bindPassword: '__IN SRECREST FILE__',
searchBase: 'ou=people,dc=theta42,dc=com',
userFilter: '(objectClass=inetOrgPerson)',
userNameAttribute: 'uid'
},
socketFile: '/var/run/proxy_lookup.socket',
redis: {
prefix: 'proxy_'
},
service:{
hostScheduler:{
enabled: true,
initial: 5000,
interval: 86400000,
}
},
};
nodejs/models/cert.js
+10 -1
View File
@@ -14,4 +14,13 @@ async function getCert(host){
}
}
module.exports = {getCert};
async function deleteCert(host){
try{
console.log('looking for', host);
return JSON.parse(await client.DEL(`${host}:latest`));
}catch(error){
return {}
}
}
module.exports = {getCert, deleteCert};
nodejs/models/dns_provider.js
+9 -5
View File
@@ -3,7 +3,7 @@
const crypto = require("crypto");
const conf = require('@simpleworkjs/conf');
const Table = require('../utils/redis_model');
const Table = require('.');
const ModelPs = require('../utils/model_pubsub');
const tldExtract = require('tld-extract').parse_host;
@@ -182,17 +182,21 @@ if(require.main === module){(async function(){try{
// console.log(await DnsProvider.findall());
let provider = await DnsProvider.get('e8443e03ac503c7b');
let provider = await DnsProvider.get('84c6613b9464fbe1');
console.log(await provider.listDomains())
// console.log(await provider.listDomains())
let domain = await Domain.get('holycore.quest') // pork
// let domain = await Domain.get('rm-rf.stream') // DO
// let domain = await Domain.get('test.wtf') // CF
// console.log(await domain.createRecord({type: 'TXT', name: 'apitewefweefwsewft222', data:'hiiiiiii'}))
console.log(await domain.createRecord({
type: 'TXT',
name: 'apitewefweefwsewft222',
data: 'sdddddddad'
}));
let txtRecords = await domain.getRecords({type: 'TXT'});
let txtRecords = await domain.getRecords();
console.log(txtRecords.map(i=>`${i.name}: ${i.data}`))
// console.log(await domain.deleteRecords({type: 'TXT'}))
nodejs/models/dns_provider/porkbun.js
+11 -5
View File
@@ -93,13 +93,19 @@ class PorkBun extends DnsApi{
});
}
async createRecord(domain, options, force){
if(force){
await this.deleteRecords(domain, options)
}
async createRecord(domain, options, force = true){
try{
// Throw errors for missing keys, do this first.
options = this.__parseOptions(options, ['type', 'name', 'data']);
// Delete the current records
if(force){
let forceOptions = new Map(Object.entries(options));
forceOptions.delete('data');
forceOptions.delete('content');
await this.deleteRecords(domain, Object.fromEntries(forceOptions));
}
let res = await this.post(`/dns/create/${domain}`, options);
return res.data.result;
nodejs/models/host.js
+18 -6
View File
@@ -2,6 +2,7 @@
const Table = require('.');
const {Domain} = require('.').models;
const {deleteCert} = require('./cert');
const ModelPs = require('../utils/model_pubsub');
const tldExtract = require('tld-extract').parse_host;
@@ -21,14 +22,18 @@ class Host extends Table{
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
'host': {isRequired: true, type: 'string', min: 3, max: 500},
'ip': {isRequired: true, type: 'string', min: 3, max: 500},
'targetPort': {isRequired: true, type: 'number', min:0, max:65535},
'forcessl': {isRequired: false, default: true, type: 'boolean'},
'targetssl': {isRequired: false, default: false, type: 'boolean'},
'is_cache': {default: false, isRequired: false, type: 'boolean',},
'is_wildcard': {default: false, isRequired: false, type: 'boolean',},
'wildcard_status': {isRequired: false, type: 'string', min: 3, max: 500},
'wildcard_matchAny': {default: false, isRequired: false, type: 'boolean',},
'wildcard_parent': {isRequired: false, type: 'string', min: 3, max: 500},
'wildcard_expires': {isRequired: false, type: 'number'},
'domain': {model: 'Domain', rel: 'one'},
@@ -76,26 +81,28 @@ class Host extends Table{
}catch(error){
console.error('bust cache error', error)
throw error;
// throw error;
}
}
static async create(data, ...args){
try{
// Validate requested host is valid host and domain
if(data.challengeType === 'DNS-01-wildcard') await this.validateWildcardCreate(data, args);
if(data.challengeType === 'DNS-01-wildcard'){
await this.validateWildcardCreate(data, args);
data.is_wildcard = true;
data.wildcard_status = "Starting"
}
// Validate requested host has a valid wildcard parent
if(data.challengeType === 'wildcardChild'){
let parentHost = await this.lookUp(data.host);
console.log('parentHost:', parentHost)
if(parentHost.is_wildcard){
data.wildcard_parent = parentHost.host;
}else{
throw new Error(`No parent wild card for ${data.host}`);
}
}
// Create the new host entry
let out = await super.create(data, ...args);
@@ -104,7 +111,7 @@ class Host extends Table{
// Fire the request for the wild card cert
// This is "back ground" job, await is intentionally missing
if(out.challengeType === 'DNS-01-wildcard') out.createWildcardCert();
if(data.challengeType === 'DNS-01-wildcard') out.createWildcardCert();
return out;
@@ -114,6 +121,7 @@ class Host extends Table{
}
static async validateWildcardCreate(data, ...args){
console.log('validateWildcardCreate here')
try{
if(!data.host.startsWith('*.')) throw new Error('not wild card');
await Domain.get(data.host);
@@ -125,6 +133,7 @@ class Host extends Table{
}
async createWildcardCert(){
console.log('createWildcardCert', this.domain)
if(!this.host.startsWith('*.')) throw new Error('not wild card');
try{
@@ -234,6 +243,7 @@ class Host extends Table{
this.createWildcardCert();
}
}catch(error){
console.error('checkWildcardForRenew instance', this.host, error)
throw error;
}
}
@@ -241,9 +251,10 @@ class Host extends Table{
static async checkWildcardForRenew(){
try{
for(let host of await this.listDetail()){
host.createWildcardCert();
host.checkWildcardForRenew();
}
}catch(error){
console.error('checkWildcardForRenew', error)
throw error;
}
}
@@ -265,6 +276,7 @@ class Host extends Table{
let out = await super.remove(...args);
await Host.buildLookUpObj();
await this.bustCache(this.host);
await deleteCert(this.host);
return out;
} catch(error){
nodejs/models/index.js
+4 -1
View File
@@ -1,6 +1,9 @@
'use strict';
const conf = require('@simpleworkjs/conf');
const {setUpTable} = require('model-redis');
const Table = setUpTable(conf.redis);
const Table = require('../utils/redis_model');
module.exports = Table;
require('./dns_provider');
nodejs/models/user_redis.js
+1 -1
View File
@@ -1,6 +1,6 @@
'use strict';
const Table = require('../utils/redis_model');
const Table = require('.');
const bcrypt = require('bcrypt');
const saltRounds = 10;
nodejs/package-lock.json
Generated
+306 -421
View File
File diff suppressed because it is too large Load Diff
nodejs/package.json
+3 -3
View File
@@ -24,7 +24,7 @@
"@popperjs/core": "^2.11.8",
"@simpleworkjs/conf": "^1.0.0",
"acme-client": "^5.4.0",
"axios": "^1.13.2",
"axios": "^1.13.5",
"bcrypt": "^6.0.0",
"bootstrap": "^5.3.8",
"ejs": "^3.1.10",
@@ -34,11 +34,11 @@
"jquery": "^3.7.1",
"ldapts": "^8.1.2",
"linux-sys-user": "^1.2.0",
"model-redis": "^0.4.0",
"model-redis": "^1.4.0",
"moment": "^2.30.1",
"mustache": "^4.2.0",
"p2psub": "^0.2.0",
"redis": "^5.10.0",
"redis": "^6.1.0",
"socket.io": "^4.8.3",
"tld-extract": "^2.1.0"
},
nodejs/routes/render.js
+2 -2
View File
@@ -25,8 +25,8 @@ frontEndModules.forEach(dep => {
// local folder.
router.use('/static', express.static(path.join(__dirname, '../public')))
router.get('/', async function(req, res, next) {
res.render('hosts', {...values});
router.get('/', (req, res) => {
res.redirect(301, '/hosts');
});
router.get('/hosts', async function(req, res, next) {
nodejs/services/host_lookup.js
+10
View File
@@ -35,6 +35,16 @@ const socket = new SocketServerJson({
// If we don't have a match, return empty object
if(!parentHost) return clientSocket.write(JSON.stringify({}));
// A wildcard host with matchAny disabled only serves subdomains that
// are explicitly defined in redis. Reaching this service means redis
// had no direct entry for the requested domain, so an inexact match
// here (the request isn't the wildcard host itself) is an undefined
// subdomain and must not be routed to the wildcard parent.
if(parentHost.is_wildcard && !parentHost.wildcard_matchAny
&& parentHost.host !== data['domain']){
return clientSocket.write(JSON.stringify({}));
}
// If the matched host belongs to a wildcard domain, set wildcard_parent
// This allows child domains to use the parent's wildcard SSL certificate
if(!parentHost.wildcard_parent){
nodejs/services/host_scheduler.js
+30 -23
View File
@@ -1,32 +1,39 @@
'use strict';
const conf = require('@simpleworkjs/conf');
const {Host} = require('../models/host');
/**
* Host Scheduler Service
*
* Manages scheduled tasks for host-related operations:
* - Wildcard SSL certificate renewal checks
*
* Schedule:
* - Initial check: 30 seconds after application starts
* - Recurring checks: Every 24 hours (86400000ms)
*
* The checkWildcardForRenew method:
* - Iterates through all hosts in the system
* - Checks if wildcard certificates are expiring within 30 days
* - Automatically renews certificates that are approaching expiration
*/
// Initial wildcard cert check 30 seconds after app starts
// Delay allows the system to fully initialize before checking certs
setTimeout(Host.checkWildcardForRenew, 30000);
function hostSchedulerService(){
/**
* Host Scheduler Service
*
* Manages scheduled tasks for host-related operations:
* - Wildcard SSL certificate renewal checks
*
* Schedule:
* - Initial check: 30 seconds after application starts
* - Recurring checks: Every 24 hours (86400000ms)
*
* The checkWildcardForRenew method:
* - Iterates through all hosts in the system
* - Checks if wildcard certificates are expiring within 30 days
* - Automatically renews certificates that are approaching expiration
*/
// Check wildcard certs once every 24 hours
// Ensures certificates are renewed well before expiration
setInterval(Host.checkWildcardForRenew, 86400000);
// Initial wildcard cert check 30 seconds after app starts
// Delay allows the system to fully initialize before checking certs
setTimeout(Host.checkWildcardForRenew.bind(Host), conf.service.hostScheduler.initial);
// Check wildcard certs once every 24 hours
// Ensures certificates are renewed well before expiration
setInterval(Host.checkWildcardForRenew.bind(Host), conf.service.hostScheduler.interval);
console.log('Host scheduler service initialized');
console.log('- Wildcard cert check: 30s after start, then every 24h');
}
if(conf.service.hostScheduler.enabled !== false) hostSchedulerService();
console.log('Host scheduler service initialized');
console.log('- Wildcard cert check: 30s after start, then every 24h');
module.exports = {};
nodejs/utils/model_pubsub.js
+70 -34
View File
@@ -1,55 +1,91 @@
'use strict';
/**
* PubSub controller dependency to handle message broadcasting
*/
const ps = require('../controller/pubsub');
function ModelPs(model){
const Model = model.constructor.name === 'Function' ? model : model.constructor
/**
* Wraps a model in a Proxy to automatically publish events on specific method calls.
* @param {Object|Function} model - The data model or instance to be proxied.
* @returns {Proxy} - The proxied model.
*/
function ModelPs(model) {
// Ensure we have a reference to the class constructor regardless of whether an instance or class was passed
const Model = model.constructor.name === 'Function' ? model : model.constructor;
function getIndex(req, res){
if(model[Model._key]) return model[Model._key];
if(req && req[Model._key]) return req[Model._key];
if(res && res[Model._key]) return res[Model._key];
/**
* Extracts the unique identifier (primary key) from the model or request/response objects.
*/
function getIndex(req, res) {
if (model[Model._key]) return model[Model._key];
if (req && req[Model._key]) return req[Model._key];
if (res && res[Model._key]) return res[Model._key];
}
function publish(prop, res, req){
try{
if(!['add', 'create', 'update', 'remove'].includes(prop)) return;
/**
* Formats and broadcasts the message via PubSub.
* Topic format: model:ClassName:Action:ID
*/
function publish(prop, res, req) {
try {
// Only trigger for specific mutation keywords
if (!['add', 'create', 'update', 'remove'].includes(prop)) return;
ps.publish(`model:${Model.name}:${prop}:${getIndex(res, req)}`, res);
}catch(error){
console.log('ModelPs.publish ERROR', error)
} catch (error) {
console.log('ModelPs.publish ERROR', error);
}
}
/**
* Standardized error logger that ignores common/non-critical HTTP errors
*/
function handleError(error, model, propKey) {
if (![401, 404, 429].includes(error.status)) {
console.error("Error PS", model.name, propKey, error);
}
}
return new Proxy(model, {
/**
* Intercepts 'new' keyword calls to ensure instances are also proxied.
*/
construct(target, args, newTarget) {
return ModelPs(Reflect.construct(target, args, newTarget))
return ModelPs(Reflect.construct(target, args, newTarget));
},
get(target, propKey, receiver) {
if(propKey == 'constructor') return target.constructor;
const targetValue = Reflect.get(target, propKey, receiver);
if (typeof targetValue === 'function') {
return function(...args){
try{
// let res = targetValue.apply(this, args); // (A)
var res = Reflect.apply(targetValue, this, args);
if(targetValue.constructor.name === 'AsyncFunction'){
res.then(function(res){
publish(propKey, res, ...args);
}).catch(function(error){
// console.error("Error PS", model.name, propKey, error)
console.log('toDo, publish errors...');
});
}else{
/**
* Intercepts property/method access.
*/
get(target, propKey, receiver) {
// Ensure constructor access remains direct
if (propKey == 'constructor') return target.constructor;
const targetValue = Reflect.get(target, propKey, receiver);
// If the property accessed is a function, wrap it to inject the PubSub logic
if (typeof targetValue === 'function') {
return function(...args) {
try {
// Execute the original method
var res = Reflect.apply(targetValue, this, args);
// Handle Asynchronous results (Promises)
if (targetValue.constructor.name === 'AsyncFunction') {
res.then(function(res) {
publish(propKey, res, ...args);
}).catch((error) => handleError(error, model, propKey));
} else {
// Handle Synchronous results
publish(propKey, res, ...args);
}
return res;
}catch(error){
// console.error("Error PS", model.name, propKey, error)
console.log("toDo, publish errors...");
} catch (error) {
handleError(error, model, propKey);
}
}
};
} else {
return targetValue;
}
@@ -57,4 +93,4 @@ function ModelPs(model){
});
}
module.exports = ModelPs;
module.exports = ModelPs;
nodejs/utils/object_validate.js
-112
View File
@@ -1,112 +0,0 @@
'use strict';
const process_type = {
number: function(key, value){
if(key.min && value < key.min) return `is to small, min ${key.min}.`
if(key.max && value > key.max) return `is to large, max ${key.max}.`
},
string: function(key, value){
if(key.min && value.length < key.min) return `is too short, min ${key.min}.`
if(key.max && value.length > key.max) return `is too short, max ${key.max}.`
},
}
function returnOrCall(value){
return typeof(value) === 'function' ? value() : value;
}
function processKeys(map, data, partial){
let errors = [];
let out = {};
for(let key of Object.keys(map)){
// Do not require "isRequired" fields for partial validation, useful for
// updates.
if(!map[key].always && partial && !data.hasOwnProperty(key)) continue;
// Make sure required keys are present
if(!partial && map[key].isRequired && !data.hasOwnProperty(key)){
errors.push({key, message:`${key} is required.`});
continue;
}
// Remove undefined keys unless they have a default option or are a
// relation
if(data[key] === undefined){
if(!map[key].default){
if(map[key].model && !map[key].type) continue;
continue;
}
}
// Check the type of the key
if(data.hasOwnProperty(key) && map[key].type && typeof(data[key]) !== map[key].type){
errors.push({key, message:`${key} is not ${map[key].type} type.`});
continue;
}
// Add the key to the process object to be returned and set any default
// if the key is blank
out[key] = data.hasOwnProperty(key) && data[key] !== undefined ? data[key] : returnOrCall(map[key].default);
// Check for type specific validations, ie: string length
if(data.hasOwnProperty(key) && process_type[map[key].type]){
let typeError = process_type[map[key].type](map[key], data[key]);
if(typeError){
errors.push({key, message:`${key} ${typeError}`});
continue;
}
}
}
// Check for errors, throw validation error if any
if(errors.length !== 0){
throw ObjectValidateError(errors);
return {__errors__: errors};
}
return out;
}
function parseFromString(map, data){
// Use the key maps data type to return string values to native
let types = {
boolean: function(value){ return value === 'false' ? false : true },
number: Number,
string: String,
object: JSON.parse
};
for(let key of Object.keys(data)){
if(map[key] && map[key].type && data[key]){
data[key] = types[map[key].type](data[key]);
}
}
return data;
}
function parseToString(data){
let types = {
object: JSON.stringify
}
return (types[typeof(data)] || String)(data);
}
function ObjectValidateError(keys, message){
let error = new Error('ObjectValidateError')
error.name = "ObjectValidateError"
error.message = message || `Invalid Keys: ${message}`
error.keys = (keys || {});
error.status = 422;
return error
}
ObjectValidateError.prototype = Error.prototype;
module.exports = {processKeys, parseFromString, ObjectValidateError, parseToString};
nodejs/utils/redis_model.js
-10
View File
@@ -1,10 +0,0 @@
'use strict';
const {setUpTable} = require('model-redis');
const conf = require('@simpleworkjs/conf');
const Table = setUpTable({
prefix: conf.redis.prefix
});
module.exports = Table;
nodejs/views/dns.ejs
+1 -1
View File
@@ -66,7 +66,7 @@
$(document).ready(async function(){
// Set the jq Templates
$.scope.providerField.put = function(){};
$.scope.DnsProvider.push(...(await app.api.get('/dns?detail=true')).results);
$.scope.DnsProvider.push(...(await app.api.get('dns?detail=true')).results);
// Populate
providerGet();
nodejs/views/hosts.ejs
+25 -2
View File
@@ -86,7 +86,7 @@
$('tr.jq-repeat-hosts').each(function(idx, el){
$(el).removeClass('table-warning');
});
$.scope.editHost.remove();
$.scope.editHost.remove(0);
}
function hostEditOpen(btn, host){
@@ -99,6 +99,8 @@
if(host.is_wildcard){
$('.hostEditPanel [name="host"]').attr('disabled', true);
// Allow toggling the wildcard matching mode when editing a wildcard host.
$('.hostEditPanel #wildcard_matchAny-container').removeClass('challengeType-container');
}
$.each(host, function( key, value ) { if(typeof value == "boolean"){
@@ -173,6 +175,7 @@
// Reset the allowed types on start
$('#challengeType-child-container').addClass('challengeType-container');
$('#challengeType-DNS-01-wildcard-container').addClass('challengeType-container');
$('#wildcard_matchAny-container').addClass('challengeType-container');
let host = $hostField.val();
@@ -180,6 +183,8 @@
// provider.
if(host.startsWith("*.") && await verifyWildcardRequirements(host)){
$('#challengeType-DNS-01-wildcard-container').removeClass('challengeType-container');
// Wildcard matching mode only applies to wildcard hosts.
$('#wildcard_matchAny-container').removeClass('challengeType-container');
return;
}
@@ -274,7 +279,7 @@
</div>
<div class="card-body">
<form class="addHost" method="PUT" action="/host/{{ host }}" onsubmit="formAJAX(this)" evalAJAX="hostEditCancle()">
<form class="addHost" method="PUT" action="host/{{ host }}" onsubmit="formAJAX(this)" evalAJAX="hostEditCancle()">
{{{ form }}}
<input type="hidden" name="edit_host" />
<button type="submit" data-type="edit" class="btn btn-warning">
@@ -363,6 +368,24 @@
</div>
</div>
<div class="form-group challengeType-container" id="wildcard_matchAny-container">
<label class="form-label">
Wildcard Matching
</label>
<div class="radio">
<label>
<input type="radio" name="wildcard_matchAny" id="wildcard_matchAny-false" value="false" checked>
Match only subdomains defined here <b>Recommended</b>
</label>
</div>
<div class="radio">
<label>
<input type="radio" name="wildcard_matchAny" id="wildcard_matchAny-true" value="true">
Match any subdomain and proxy to this host
</label>
</div>
</div>
<div class="mb-3 form-group">
<label for="ip" class="form-label">
Target IP or Host Name
ops/nginx_conf/autossl.conf
+2 -2
View File
@@ -1,9 +1,9 @@
listen 443 ssl http2;
listen 4443 ssl;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!3DES:!MD5;
ssl_certificate_by_lua_block {
auto_ssl:ssl_certificate()
ops/nginx_conf/targetinfo.lua
+8
View File
@@ -37,6 +37,14 @@ function M.get(ngx, domain, targetInfo)
local res, err = red:hgetall("proxy_Host_"..domain)
res = red:array_to_hash(res)
-- Return the connection to the pool instead of closing it, so it can be
-- reused by later requests. Without this a new connection is opened per
-- request and never released, exhausting sockets under load.
local ok, err = red:set_keepalive(10000, 100)
if not ok then
ngx.log(ngx.ERR, "failed to set redis keepalive: ", err)
end
if not res["ip"] then
if connect("/var/run/proxy_lookup.socket") then
local socket = require("socket.unix")()